diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000000..d4f21a2e7ac8ca --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @anza-xyz/backport-reviewers @anza-xyz/releng diff --git a/CHANGELOG.md b/CHANGELOG.md index 9303c74e6c6d32..73e1cc50a7836a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ Release channels have their own copy of this changelog: * New program deployments default to the exact size of a program, instead of double the size. Program accounts must be extended with `solana program extend` before an upgrade if they need to accommodate larger programs. - * CLI: Can specify `--with-compute-unit-price` and `--max-sign-attempts` during program deployment + * CLI: Can specify `--with-compute-unit-price`, `--max-sign-attempts`, and `--use-rpc` during program deployment * Upgrade Notes * `solana-program` and `solana-sdk` default to support for Borsh v1, with limited backward compatibility for v0.10 and v0.9. Please upgrade to Borsh v1. diff --git a/Cargo.lock b/Cargo.lock index 1a06894ff8c373..0b0b582ff6e7c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,7 +64,7 @@ dependencies = [ [[package]] name = "agave-cargo-registry" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "flate2", @@ -93,7 +93,7 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-sdk", @@ -103,7 +103,7 @@ dependencies = [ [[package]] name = "agave-install" -version = "1.18.11" +version = "1.18.12" dependencies = [ "atty", "bincode", @@ -138,7 +138,7 @@ dependencies = [ [[package]] name = "agave-ledger-tool" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_cmd", "bs58", @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "agave-validator" -version = "1.18.11" +version = "1.18.12" dependencies = [ "agave-geyser-plugin-interface", "chrono", @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "agave-watchtower" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "humantime", @@ -2371,7 +2371,7 @@ dependencies = [ [[package]] name = "gen-headers" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "regex", @@ -2379,7 +2379,7 @@ dependencies = [ [[package]] name = "gen-syscall-list" -version = "1.18.11" +version = "1.18.12" dependencies = [ "regex", ] @@ -4361,7 +4361,7 @@ dependencies = [ [[package]] name = "proto" -version = "1.18.11" +version = "1.18.12" dependencies = [ "protobuf-src", "tonic-build", @@ -4604,7 +4604,7 @@ dependencies = [ [[package]] name = "rbpf-cli" -version = "1.18.11" +version = "1.18.12" [[package]] name = "rcgen" @@ -4882,9 +4882,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring 0.17.3", @@ -5429,7 +5429,7 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "assert_matches", @@ -5454,7 +5454,7 @@ dependencies = [ [[package]] name = "solana-accounts-bench" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "log", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "solana-accounts-cluster-bench" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "log", @@ -5498,7 +5498,7 @@ dependencies = [ [[package]] name = "solana-accounts-db" -version = "1.18.11" +version = "1.18.12" dependencies = [ "arrayref", "assert_matches", @@ -5565,7 +5565,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bytemuck", @@ -5584,7 +5584,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program-tests" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5595,7 +5595,7 @@ dependencies = [ [[package]] name = "solana-banking-bench" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 3.2.23", "crossbeam-channel", @@ -5619,7 +5619,7 @@ dependencies = [ [[package]] name = "solana-banks-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "borsh 1.2.1", "futures 0.3.30", @@ -5636,7 +5636,7 @@ dependencies = [ [[package]] name = "solana-banks-interface" -version = "1.18.11" +version = "1.18.12" dependencies = [ "serde", "solana-sdk", @@ -5645,7 +5645,7 @@ dependencies = [ [[package]] name = "solana-banks-server" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "crossbeam-channel", @@ -5663,7 +5663,7 @@ dependencies = [ [[package]] name = "solana-bench-streamer" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 3.2.23", "crossbeam-channel", @@ -5674,7 +5674,7 @@ dependencies = [ [[package]] name = "solana-bench-tps" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "crossbeam-channel", @@ -5715,7 +5715,7 @@ dependencies = [ [[package]] name = "solana-bloom" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bv", "fnv", @@ -5732,7 +5732,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5753,7 +5753,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program-tests" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5764,7 +5764,7 @@ dependencies = [ [[package]] name = "solana-bucket-map" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bv", "bytemuck", @@ -5783,7 +5783,7 @@ dependencies = [ [[package]] name = "solana-cargo-build-bpf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-logger", @@ -5791,7 +5791,7 @@ dependencies = [ [[package]] name = "solana-cargo-build-sbf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_cmd", "bzip2", @@ -5812,11 +5812,11 @@ dependencies = [ [[package]] name = "solana-cargo-test-bpf" -version = "1.18.11" +version = "1.18.12" [[package]] name = "solana-cargo-test-sbf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "cargo_metadata", "clap 3.2.23", @@ -5827,7 +5827,7 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "chrono", @@ -5844,7 +5844,7 @@ dependencies = [ [[package]] name = "solana-clap-v3-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "chrono", @@ -5862,7 +5862,7 @@ dependencies = [ [[package]] name = "solana-cli" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5917,7 +5917,7 @@ dependencies = [ [[package]] name = "solana-cli-config" -version = "1.18.11" +version = "1.18.12" dependencies = [ "anyhow", "dirs-next", @@ -5932,7 +5932,7 @@ dependencies = [ [[package]] name = "solana-cli-output" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "base64 0.21.7", @@ -5958,7 +5958,7 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -5990,7 +5990,7 @@ dependencies = [ [[package]] name = "solana-client-test" -version = "1.18.11" +version = "1.18.12" dependencies = [ "futures-util", "rand 0.8.5", @@ -6020,7 +6020,7 @@ dependencies = [ [[package]] name = "solana-compute-budget-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program-runtime", "solana-sdk", @@ -6028,7 +6028,7 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "chrono", @@ -6041,7 +6041,7 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -6065,7 +6065,7 @@ dependencies = [ [[package]] name = "solana-core" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6151,7 +6151,7 @@ dependencies = [ [[package]] name = "solana-cost-model" -version = "1.18.11" +version = "1.18.12" dependencies = [ "lazy_static", "log", @@ -6176,7 +6176,7 @@ dependencies = [ [[package]] name = "solana-dos" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "clap 3.2.23", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "solana-download-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "console", "indicatif", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "solana-ed25519-program-tests" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "ed25519-dalek", @@ -6229,7 +6229,7 @@ dependencies = [ [[package]] name = "solana-entry" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -6251,7 +6251,7 @@ dependencies = [ [[package]] name = "solana-faucet" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "byteorder", @@ -6273,7 +6273,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bitflags 2.4.2", "block-buffer 0.10.4", @@ -6298,7 +6298,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.18.11" +version = "1.18.12" dependencies = [ "proc-macro2", "quote", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "solana-genesis" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -6333,7 +6333,7 @@ dependencies = [ [[package]] name = "solana-genesis-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-accounts-db", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-manager" -version = "1.18.11" +version = "1.18.12" dependencies = [ "agave-geyser-plugin-interface", "bs58", @@ -6369,7 +6369,7 @@ dependencies = [ [[package]] name = "solana-gossip" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "solana-keygen" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bs58", "clap 3.2.23", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "solana-ledger" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "solana-loader-v4-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "solana-local-cluster" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "crossbeam-channel", @@ -6557,7 +6557,7 @@ dependencies = [ [[package]] name = "solana-log-analyzer" -version = "1.18.11" +version = "1.18.12" dependencies = [ "byte-unit", "clap 3.2.23", @@ -6569,7 +6569,7 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.18.11" +version = "1.18.12" dependencies = [ "env_logger", "lazy_static", @@ -6578,7 +6578,7 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-sdk", @@ -6586,11 +6586,11 @@ dependencies = [ [[package]] name = "solana-memory-management" -version = "1.18.11" +version = "1.18.12" [[package]] name = "solana-merkle-root-bench" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "log", @@ -6603,7 +6603,7 @@ dependencies = [ [[package]] name = "solana-merkle-tree" -version = "1.18.11" +version = "1.18.12" dependencies = [ "fast-math", "hex", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "env_logger", @@ -6628,7 +6628,7 @@ dependencies = [ [[package]] name = "solana-net-shaper" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 3.2.23", "rand 0.8.5", @@ -6639,7 +6639,7 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "clap 3.2.23", @@ -6665,7 +6665,7 @@ checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" [[package]] name = "solana-notifier" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "reqwest", @@ -6675,7 +6675,7 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "ahash 0.8.7", "assert_matches", @@ -6706,7 +6706,7 @@ dependencies = [ [[package]] name = "solana-poh" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -6728,7 +6728,7 @@ dependencies = [ [[package]] name = "solana-poh-bench" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 3.2.23", "log", @@ -6743,7 +6743,7 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "anyhow", "ark-bn254", @@ -6801,7 +6801,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6830,7 +6830,7 @@ dependencies = [ [[package]] name = "solana-program-test" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "async-trait", @@ -6859,7 +6859,7 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "anyhow", "crossbeam-channel", @@ -6883,7 +6883,7 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-mutex", "async-trait", @@ -6911,7 +6911,7 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.18.11" +version = "1.18.12" dependencies = [ "lazy_static", "num_cpus", @@ -6919,7 +6919,7 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "console", @@ -6938,7 +6938,7 @@ dependencies = [ [[package]] name = "solana-rpc" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -6997,7 +6997,7 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "async-trait", @@ -7026,7 +7026,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bs58", @@ -7046,7 +7046,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "anyhow", "clap 2.33.3", @@ -7063,7 +7063,7 @@ dependencies = [ [[package]] name = "solana-rpc-test" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bs58", @@ -7090,7 +7090,7 @@ dependencies = [ [[package]] name = "solana-runtime" -version = "1.18.11" +version = "1.18.12" dependencies = [ "aquamarine", "arrayref", @@ -7173,7 +7173,7 @@ dependencies = [ [[package]] name = "solana-runtime-transaction" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -7187,7 +7187,7 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.18.11" +version = "1.18.12" dependencies = [ "anyhow", "assert_matches", @@ -7246,7 +7246,7 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bs58", "proc-macro2", @@ -7263,7 +7263,7 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-send-transaction-service" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "log", @@ -7278,7 +7278,7 @@ dependencies = [ [[package]] name = "solana-stake-accounts" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "solana-clap-utils", @@ -7294,7 +7294,7 @@ dependencies = [ [[package]] name = "solana-stake-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -7311,7 +7311,7 @@ dependencies = [ [[package]] name = "solana-storage-bigtable" -version = "1.18.11" +version = "1.18.12" dependencies = [ "backoff", "bincode", @@ -7343,7 +7343,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bs58", @@ -7359,7 +7359,7 @@ dependencies = [ [[package]] name = "solana-store-tool" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "log", @@ -7371,7 +7371,7 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "async-channel", @@ -7404,7 +7404,7 @@ dependencies = [ [[package]] name = "solana-system-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -7418,7 +7418,7 @@ dependencies = [ [[package]] name = "solana-test-validator" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -7448,7 +7448,7 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -7462,7 +7462,7 @@ dependencies = [ [[package]] name = "solana-tokens" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -7495,7 +7495,7 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -7517,7 +7517,7 @@ dependencies = [ [[package]] name = "solana-transaction-dos" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "clap 2.33.3", @@ -7544,7 +7544,7 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "base64 0.21.7", @@ -7567,7 +7567,7 @@ dependencies = [ [[package]] name = "solana-turbine" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -7605,7 +7605,7 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "solana-connection-cache", @@ -7618,11 +7618,11 @@ dependencies = [ [[package]] name = "solana-unified-scheduler-logic" -version = "1.18.11" +version = "1.18.12" [[package]] name = "solana-unified-scheduler-pool" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "solana-ledger", @@ -7636,7 +7636,7 @@ dependencies = [ [[package]] name = "solana-upload-perf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "serde_json", "solana-metrics", @@ -7644,7 +7644,7 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "rustc_version 0.4.0", @@ -7658,7 +7658,7 @@ dependencies = [ [[package]] name = "solana-vote" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "crossbeam-channel", @@ -7677,7 +7677,7 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -7700,7 +7700,7 @@ dependencies = [ [[package]] name = "solana-wen-restart" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "prost", @@ -7722,7 +7722,7 @@ dependencies = [ [[package]] name = "solana-zk-keygen" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bs58", "clap 3.2.23", @@ -7741,7 +7741,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bytemuck", "criterion", @@ -7755,7 +7755,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program-tests" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bytemuck", "curve25519-dalek", @@ -7767,7 +7767,7 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.18.11" +version = "1.18.12" dependencies = [ "aes-gcm-siv", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index 75eac661051b4a..fac87078a28872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ exclude = ["programs/sbf"] resolver = "2" [workspace.package] -version = "1.18.11" +version = "1.18.12" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" homepage = "https://solanalabs.com/" @@ -287,7 +287,7 @@ reqwest = { version = "0.11.23", default-features = false } rolling-file = "0.2.0" rpassword = "7.3" rustc_version = "0.4" -rustls = { version = "0.21.10", default-features = false, features = ["quic"] } +rustls = { version = "0.21.11", default-features = false, features = ["quic"] } rustversion = "1.0.14" scopeguard = "1.2.0" semver = "1.0.21" @@ -307,86 +307,86 @@ smallvec = "1.13.2" smpl_jwt = "0.7.1" socket2 = "0.5.5" soketto = "0.7" -solana-account-decoder = { path = "account-decoder", version = "=1.18.11" } -solana-accounts-db = { path = "accounts-db", version = "=1.18.11" } -solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=1.18.11" } -solana-banks-client = { path = "banks-client", version = "=1.18.11" } -solana-banks-interface = { path = "banks-interface", version = "=1.18.11" } -solana-banks-server = { path = "banks-server", version = "=1.18.11" } -solana-bench-tps = { path = "bench-tps", version = "=1.18.11" } -solana-bloom = { path = "bloom", version = "=1.18.11" } -solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.18.11" } -solana-bucket-map = { path = "bucket_map", version = "=1.18.11" } -agave-cargo-registry = { path = "cargo-registry", version = "=1.18.11" } -solana-clap-utils = { path = "clap-utils", version = "=1.18.11" } -solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.18.11" } -solana-cli = { path = "cli", version = "=1.18.11" } -solana-cli-config = { path = "cli-config", version = "=1.18.11" } -solana-cli-output = { path = "cli-output", version = "=1.18.11" } -solana-client = { path = "client", version = "=1.18.11" } -solana-compute-budget-program = { path = "programs/compute-budget", version = "=1.18.11" } -solana-config-program = { path = "programs/config", version = "=1.18.11" } -solana-connection-cache = { path = "connection-cache", version = "=1.18.11", default-features = false } -solana-core = { path = "core", version = "=1.18.11" } -solana-cost-model = { path = "cost-model", version = "=1.18.11" } -solana-download-utils = { path = "download-utils", version = "=1.18.11" } -solana-entry = { path = "entry", version = "=1.18.11" } -solana-faucet = { path = "faucet", version = "=1.18.11" } -solana-frozen-abi = { path = "frozen-abi", version = "=1.18.11" } -solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.18.11" } -solana-genesis = { path = "genesis", version = "=1.18.11" } -solana-genesis-utils = { path = "genesis-utils", version = "=1.18.11" } -agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.18.11" } -solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.18.11" } -solana-gossip = { path = "gossip", version = "=1.18.11" } -solana-ledger = { path = "ledger", version = "=1.18.11" } -solana-loader-v4-program = { path = "programs/loader-v4", version = "=1.18.11" } -solana-local-cluster = { path = "local-cluster", version = "=1.18.11" } -solana-logger = { path = "logger", version = "=1.18.11" } -solana-measure = { path = "measure", version = "=1.18.11" } -solana-merkle-tree = { path = "merkle-tree", version = "=1.18.11" } -solana-metrics = { path = "metrics", version = "=1.18.11" } -solana-net-utils = { path = "net-utils", version = "=1.18.11" } +solana-account-decoder = { path = "account-decoder", version = "=1.18.12" } +solana-accounts-db = { path = "accounts-db", version = "=1.18.12" } +solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=1.18.12" } +solana-banks-client = { path = "banks-client", version = "=1.18.12" } +solana-banks-interface = { path = "banks-interface", version = "=1.18.12" } +solana-banks-server = { path = "banks-server", version = "=1.18.12" } +solana-bench-tps = { path = "bench-tps", version = "=1.18.12" } +solana-bloom = { path = "bloom", version = "=1.18.12" } +solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.18.12" } +solana-bucket-map = { path = "bucket_map", version = "=1.18.12" } +agave-cargo-registry = { path = "cargo-registry", version = "=1.18.12" } +solana-clap-utils = { path = "clap-utils", version = "=1.18.12" } +solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.18.12" } +solana-cli = { path = "cli", version = "=1.18.12" } +solana-cli-config = { path = "cli-config", version = "=1.18.12" } +solana-cli-output = { path = "cli-output", version = "=1.18.12" } +solana-client = { path = "client", version = "=1.18.12" } +solana-compute-budget-program = { path = "programs/compute-budget", version = "=1.18.12" } +solana-config-program = { path = "programs/config", version = "=1.18.12" } +solana-connection-cache = { path = "connection-cache", version = "=1.18.12", default-features = false } +solana-core = { path = "core", version = "=1.18.12" } +solana-cost-model = { path = "cost-model", version = "=1.18.12" } +solana-download-utils = { path = "download-utils", version = "=1.18.12" } +solana-entry = { path = "entry", version = "=1.18.12" } +solana-faucet = { path = "faucet", version = "=1.18.12" } +solana-frozen-abi = { path = "frozen-abi", version = "=1.18.12" } +solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.18.12" } +solana-genesis = { path = "genesis", version = "=1.18.12" } +solana-genesis-utils = { path = "genesis-utils", version = "=1.18.12" } +agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.18.12" } +solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.18.12" } +solana-gossip = { path = "gossip", version = "=1.18.12" } +solana-ledger = { path = "ledger", version = "=1.18.12" } +solana-loader-v4-program = { path = "programs/loader-v4", version = "=1.18.12" } +solana-local-cluster = { path = "local-cluster", version = "=1.18.12" } +solana-logger = { path = "logger", version = "=1.18.12" } +solana-measure = { path = "measure", version = "=1.18.12" } +solana-merkle-tree = { path = "merkle-tree", version = "=1.18.12" } +solana-metrics = { path = "metrics", version = "=1.18.12" } +solana-net-utils = { path = "net-utils", version = "=1.18.12" } solana-nohash-hasher = "0.2.1" -solana-notifier = { path = "notifier", version = "=1.18.11" } -solana-perf = { path = "perf", version = "=1.18.11" } -solana-poh = { path = "poh", version = "=1.18.11" } -solana-program = { path = "sdk/program", version = "=1.18.11" } -solana-program-runtime = { path = "program-runtime", version = "=1.18.11" } -solana-program-test = { path = "program-test", version = "=1.18.11" } -solana-pubsub-client = { path = "pubsub-client", version = "=1.18.11" } -solana-quic-client = { path = "quic-client", version = "=1.18.11" } -solana-rayon-threadlimit = { path = "rayon-threadlimit", version = "=1.18.11" } -solana-remote-wallet = { path = "remote-wallet", version = "=1.18.11", default-features = false } -solana-unified-scheduler-logic = { path = "unified-scheduler-logic", version = "=1.18.11" } -solana-unified-scheduler-pool = { path = "unified-scheduler-pool", version = "=1.18.11" } -solana-rpc = { path = "rpc", version = "=1.18.11" } -solana-rpc-client = { path = "rpc-client", version = "=1.18.11", default-features = false } -solana-rpc-client-api = { path = "rpc-client-api", version = "=1.18.11" } -solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=1.18.11" } -solana-runtime = { path = "runtime", version = "=1.18.11" } -solana-runtime-transaction = { path = "runtime-transaction", version = "=1.18.11" } -solana-sdk = { path = "sdk", version = "=1.18.11" } -solana-sdk-macro = { path = "sdk/macro", version = "=1.18.11" } -solana-send-transaction-service = { path = "send-transaction-service", version = "=1.18.11" } -solana-stake-program = { path = "programs/stake", version = "=1.18.11" } -solana-storage-bigtable = { path = "storage-bigtable", version = "=1.18.11" } -solana-storage-proto = { path = "storage-proto", version = "=1.18.11" } -solana-streamer = { path = "streamer", version = "=1.18.11" } -solana-system-program = { path = "programs/system", version = "=1.18.11" } -solana-test-validator = { path = "test-validator", version = "=1.18.11" } -solana-thin-client = { path = "thin-client", version = "=1.18.11" } -solana-tpu-client = { path = "tpu-client", version = "=1.18.11", default-features = false } -solana-transaction-status = { path = "transaction-status", version = "=1.18.11" } -solana-turbine = { path = "turbine", version = "=1.18.11" } -solana-udp-client = { path = "udp-client", version = "=1.18.11" } -solana-version = { path = "version", version = "=1.18.11" } -solana-vote = { path = "vote", version = "=1.18.11" } -solana-vote-program = { path = "programs/vote", version = "=1.18.11" } -solana-wen-restart = { path = "wen-restart", version = "=1.18.11" } -solana-zk-keygen = { path = "zk-keygen", version = "=1.18.11" } -solana-zk-token-proof-program = { path = "programs/zk-token-proof", version = "=1.18.11" } -solana-zk-token-sdk = { path = "zk-token-sdk", version = "=1.18.11" } +solana-notifier = { path = "notifier", version = "=1.18.12" } +solana-perf = { path = "perf", version = "=1.18.12" } +solana-poh = { path = "poh", version = "=1.18.12" } +solana-program = { path = "sdk/program", version = "=1.18.12" } +solana-program-runtime = { path = "program-runtime", version = "=1.18.12" } +solana-program-test = { path = "program-test", version = "=1.18.12" } +solana-pubsub-client = { path = "pubsub-client", version = "=1.18.12" } +solana-quic-client = { path = "quic-client", version = "=1.18.12" } +solana-rayon-threadlimit = { path = "rayon-threadlimit", version = "=1.18.12" } +solana-remote-wallet = { path = "remote-wallet", version = "=1.18.12", default-features = false } +solana-unified-scheduler-logic = { path = "unified-scheduler-logic", version = "=1.18.12" } +solana-unified-scheduler-pool = { path = "unified-scheduler-pool", version = "=1.18.12" } +solana-rpc = { path = "rpc", version = "=1.18.12" } +solana-rpc-client = { path = "rpc-client", version = "=1.18.12", default-features = false } +solana-rpc-client-api = { path = "rpc-client-api", version = "=1.18.12" } +solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=1.18.12" } +solana-runtime = { path = "runtime", version = "=1.18.12" } +solana-runtime-transaction = { path = "runtime-transaction", version = "=1.18.12" } +solana-sdk = { path = "sdk", version = "=1.18.12" } +solana-sdk-macro = { path = "sdk/macro", version = "=1.18.12" } +solana-send-transaction-service = { path = "send-transaction-service", version = "=1.18.12" } +solana-stake-program = { path = "programs/stake", version = "=1.18.12" } +solana-storage-bigtable = { path = "storage-bigtable", version = "=1.18.12" } +solana-storage-proto = { path = "storage-proto", version = "=1.18.12" } +solana-streamer = { path = "streamer", version = "=1.18.12" } +solana-system-program = { path = "programs/system", version = "=1.18.12" } +solana-test-validator = { path = "test-validator", version = "=1.18.12" } +solana-thin-client = { path = "thin-client", version = "=1.18.12" } +solana-tpu-client = { path = "tpu-client", version = "=1.18.12", default-features = false } +solana-transaction-status = { path = "transaction-status", version = "=1.18.12" } +solana-turbine = { path = "turbine", version = "=1.18.12" } +solana-udp-client = { path = "udp-client", version = "=1.18.12" } +solana-version = { path = "version", version = "=1.18.12" } +solana-vote = { path = "vote", version = "=1.18.12" } +solana-vote-program = { path = "programs/vote", version = "=1.18.12" } +solana-wen-restart = { path = "wen-restart", version = "=1.18.12" } +solana-zk-keygen = { path = "zk-keygen", version = "=1.18.12" } +solana-zk-token-proof-program = { path = "programs/zk-token-proof", version = "=1.18.12" } +solana-zk-token-sdk = { path = "zk-token-sdk", version = "=1.18.12" } solana_rbpf = "=0.8.0" spl-associated-token-account = "=2.3.0" spl-instruction-padding = "0.1" diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 71f215fa01dc9e..5ed223bbbbc9bf 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -7837,7 +7837,10 @@ impl AccountsDb { Some((*loaded_account.pubkey(), loaded_account.loaded_hash())) }, |accum: &DashMap, loaded_account: LoadedAccount| { - let loaded_hash = loaded_account.loaded_hash(); + let mut loaded_hash = loaded_account.loaded_hash(); + if loaded_hash == AccountHash(Hash::default()) { + loaded_hash = Self::hash_account(&loaded_account, loaded_account.pubkey()) + } accum.insert(*loaded_account.pubkey(), loaded_hash); }, ); @@ -7869,9 +7872,13 @@ impl AccountsDb { |accum: &DashMap, loaded_account: LoadedAccount| { // Storage may have duplicates so only keep the latest version for each key + let mut loaded_hash = loaded_account.loaded_hash(); + if loaded_hash == AccountHash(Hash::default()) { + loaded_hash = Self::hash_account(&loaded_account, loaded_account.pubkey()) + } accum.insert( *loaded_account.pubkey(), - (loaded_account.loaded_hash(), loaded_account.take_account()), + (loaded_hash, loaded_account.take_account()), ); }, ); diff --git a/cli/src/program.rs b/cli/src/program.rs index 2cb2cd4289bbdf..2c96ae266dc132 100644 --- a/cli/src/program.rs +++ b/cli/src/program.rs @@ -99,6 +99,7 @@ pub enum ProgramCliCommand { skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, }, Upgrade { fee_payer_signer_index: SignerIndex, @@ -119,6 +120,7 @@ pub enum ProgramCliCommand { skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, }, SetBufferAuthority { buffer_pubkey: Pubkey, @@ -262,6 +264,9 @@ impl ProgramSubCommands for App<'_, '_> { whichever comes first.", ), ) + .arg(Arg::with_name("use_rpc").long("use-rpc").help( + "Send write transactions to the configured RPC instead of validator TPUs", + )) .arg(compute_unit_price_arg()), ) .subcommand( @@ -355,6 +360,9 @@ impl ProgramSubCommands for App<'_, '_> { whichever comes first.", ), ) + .arg(Arg::with_name("use_rpc").long("use-rpc").help( + "Send transactions to the configured RPC instead of validator TPUs", + )) .arg(compute_unit_price_arg()), ) .subcommand( @@ -667,6 +675,7 @@ pub fn parse_program_subcommand( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc: matches.is_present("use_rpc"), }), signers: signer_info.signers, } @@ -755,6 +764,7 @@ pub fn parse_program_subcommand( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc: matches.is_present("use_rpc"), }), signers: signer_info.signers, } @@ -949,6 +959,7 @@ pub fn process_program_subcommand( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc, } => process_program_deploy( rpc_client, config, @@ -965,6 +976,7 @@ pub fn process_program_subcommand( *skip_fee_check, *compute_unit_price, *max_sign_attempts, + *use_rpc, ), ProgramCliCommand::Upgrade { fee_payer_signer_index, @@ -995,6 +1007,7 @@ pub fn process_program_subcommand( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc, } => process_write_buffer( rpc_client, config, @@ -1007,6 +1020,7 @@ pub fn process_program_subcommand( *skip_fee_check, *compute_unit_price, *max_sign_attempts, + *use_rpc, ), ProgramCliCommand::SetBufferAuthority { buffer_pubkey, @@ -1125,6 +1139,7 @@ fn process_program_deploy( skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, ) -> ProcessResult { let fee_payer_signer = config.signers[fee_payer_signer_index]; let upgrade_authority_signer = config.signers[upgrade_authority_signer_index]; @@ -1266,6 +1281,7 @@ fn process_program_deploy( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc, ) } else { do_process_program_upgrade( @@ -1282,6 +1298,7 @@ fn process_program_deploy( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc, ) }; if result.is_ok() && is_final { @@ -1428,6 +1445,7 @@ fn process_write_buffer( skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, ) -> ProcessResult { let fee_payer_signer = config.signers[fee_payer_signer_index]; let buffer_authority = config.signers[buffer_authority_signer_index]; @@ -1495,6 +1513,7 @@ fn process_write_buffer( skip_fee_check, compute_unit_price, max_sign_attempts, + use_rpc, ); if result.is_err() && buffer_signer_index.is_none() && buffer_signer.is_some() { report_ephemeral_mnemonic(words, mnemonic); @@ -2222,6 +2241,7 @@ fn do_process_program_write_and_deploy( skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, ) -> ProcessResult { let blockhash = rpc_client.get_latest_blockhash()?; @@ -2361,6 +2381,7 @@ fn do_process_program_write_and_deploy( Some(buffer_authority_signer), program_signers, max_sign_attempts, + use_rpc, )?; if let Some(program_signers) = program_signers { @@ -2391,6 +2412,7 @@ fn do_process_program_upgrade( skip_fee_check: bool, compute_unit_price: Option, max_sign_attempts: usize, + use_rpc: bool, ) -> ProcessResult { let blockhash = rpc_client.get_latest_blockhash()?; @@ -2509,6 +2531,7 @@ fn do_process_program_upgrade( Some(upgrade_authority), Some(&[upgrade_authority]), max_sign_attempts, + use_rpc, )?; let program_id = CliProgramId { @@ -2703,6 +2726,7 @@ fn send_deploy_messages( write_signer: Option<&dyn Signer>, final_signers: Option<&[&dyn Signer]>, max_sign_attempts: usize, + use_rpc: bool, ) -> Result<(), Box> { if let Some(message) = initial_message { if let Some(initial_signer) = initial_signer { @@ -2779,14 +2803,15 @@ fn send_deploy_messages( solana_client::tpu_client::TpuClientConfig::default(), cache, ); - let tpu_client = rpc_client + let tpu_client = (!use_rpc).then(|| rpc_client .runtime() .block_on(tpu_client_fut) - .expect("Should return a valid tpu client"); + .expect("Should return a valid tpu client") + ); send_and_confirm_transactions_in_parallel_blocking( rpc_client.clone(), - Some(tpu_client), + tpu_client, &write_messages, &[fee_payer_signer, write_signer], SendAndConfirmConfig { @@ -2939,6 +2964,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -2969,6 +2995,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -3001,6 +3028,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3035,6 +3063,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -3068,6 +3097,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3104,6 +3134,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3136,6 +3167,7 @@ mod tests { allow_excessive_balance: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![Box::new(read_keypair_file(&keypair_file).unwrap())], } @@ -3166,6 +3198,37 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 1, + use_rpc: false, + }), + signers: vec![Box::new(read_keypair_file(&keypair_file).unwrap())], + } + ); + + let test_command = test_commands.clone().get_matches_from(vec![ + "test", + "program", + "deploy", + "/Users/test/program.so", + "--use-rpc", + ]); + assert_eq!( + parse_command(&test_command, &default_signer, &mut None).unwrap(), + CliCommandInfo { + command: CliCommand::Program(ProgramCliCommand::Deploy { + program_location: Some("/Users/test/program.so".to_string()), + fee_payer_signer_index: 0, + buffer_signer_index: None, + buffer_pubkey: None, + program_signer_index: None, + program_pubkey: None, + upgrade_authority_signer_index: 0, + is_final: false, + max_len: None, + allow_excessive_balance: false, + skip_fee_check: false, + compute_unit_price: None, + max_sign_attempts: 5, + use_rpc: true, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -3202,6 +3265,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -3229,6 +3293,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![read_keypair_file(&keypair_file).unwrap().into()], } @@ -3259,6 +3324,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3292,6 +3358,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3330,6 +3397,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![ read_keypair_file(&keypair_file).unwrap().into(), @@ -3361,6 +3429,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 10, + use_rpc: false, }), signers: vec![Box::new(read_keypair_file(&keypair_file).unwrap())], } @@ -3893,6 +3962,7 @@ mod tests { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }), signers: vec![&default_keypair], output_format: OutputFormat::JsonCompact, diff --git a/cli/tests/program.rs b/cli/tests/program.rs index c67f6af37ad176..0365f3b5335ee8 100644 --- a/cli/tests/program.rs +++ b/cli/tests/program.rs @@ -99,6 +99,7 @@ fn test_cli_program_deploy_non_upgradeable() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -147,6 +148,7 @@ fn test_cli_program_deploy_non_upgradeable() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); let account1 = rpc_client @@ -204,6 +206,7 @@ fn test_cli_program_deploy_non_upgradeable() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let err = process_command(&config).unwrap_err(); assert_eq!( @@ -229,6 +232,7 @@ fn test_cli_program_deploy_non_upgradeable() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap_err(); } @@ -292,6 +296,7 @@ fn test_cli_program_deploy_no_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -321,6 +326,7 @@ fn test_cli_program_deploy_no_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap_err(); } @@ -385,6 +391,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -436,6 +443,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let response = process_command(&config); let json: Value = serde_json::from_str(&response.unwrap()).unwrap(); @@ -481,6 +489,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); let program_account = rpc_client.get_account(&program_pubkey).unwrap(); @@ -539,6 +548,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); let program_account = rpc_client.get_account(&program_pubkey).unwrap(); @@ -617,6 +627,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap_err(); @@ -636,6 +647,7 @@ fn test_cli_program_deploy_with_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let response = process_command(&config); let json: Value = serde_json::from_str(&response.unwrap()).unwrap(); @@ -742,6 +754,7 @@ fn test_cli_program_close_program() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; process_command(&config).unwrap(); @@ -854,6 +867,7 @@ fn test_cli_program_extend_program() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; process_command(&config).unwrap(); @@ -903,6 +917,7 @@ fn test_cli_program_extend_program() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap_err(); @@ -937,6 +952,7 @@ fn test_cli_program_extend_program() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); } @@ -1003,6 +1019,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -1041,6 +1058,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let response = process_command(&config); let json: Value = serde_json::from_str(&response.unwrap()).unwrap(); @@ -1106,6 +1124,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let response = process_command(&config); let json: Value = serde_json::from_str(&response.unwrap()).unwrap(); @@ -1147,6 +1166,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); let response = process_command(&config); let json: Value = serde_json::from_str(&response.unwrap()).unwrap(); @@ -1224,6 +1244,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -1268,6 +1289,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); config.signers = vec![&keypair, &buffer_keypair]; @@ -1285,6 +1307,7 @@ fn test_cli_program_write_buffer() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let error = process_command(&config).unwrap_err(); @@ -1346,6 +1369,7 @@ fn test_cli_program_set_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap(); @@ -1400,6 +1424,7 @@ fn test_cli_program_set_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; process_command(&config).unwrap_err(); @@ -1447,6 +1472,7 @@ fn test_cli_program_set_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; process_command(&config).unwrap(); @@ -1505,6 +1531,7 @@ fn test_cli_program_mismatch_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap(); @@ -1531,6 +1558,7 @@ fn test_cli_program_mismatch_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap_err(); @@ -1550,6 +1578,7 @@ fn test_cli_program_mismatch_buffer_authority() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); } @@ -1635,6 +1664,7 @@ fn test_cli_program_deploy_with_offline_signing(use_offline_signer_as_fee_payer: skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; process_command(&config).unwrap(); @@ -1804,6 +1834,7 @@ fn test_cli_program_show() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); @@ -1867,6 +1898,7 @@ fn test_cli_program_show() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); config.output_format = OutputFormat::JsonCompact; let min_slot = rpc_client.get_slot().unwrap(); @@ -1997,6 +2029,7 @@ fn test_cli_program_dump() { skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(&config).unwrap(); @@ -2042,6 +2075,7 @@ fn create_buffer_with_offline_authority<'a>( skip_fee_check: false, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, }); process_command(config).unwrap(); let buffer_account = rpc_client.get_account(&buffer_signer.pubkey()).unwrap(); @@ -2069,7 +2103,10 @@ fn create_buffer_with_offline_authority<'a>( } #[allow(clippy::assertions_on_constants)] -fn cli_program_deploy_with_args(compute_unit_price: Option) { +#[test_case(None, false; "default")] +#[test_case(Some(10), false; "with_compute_unit_price")] +#[test_case(None, true; "use_rpc")] +fn test_cli_program_deploy_with_args(compute_unit_price: Option, use_rpc: bool) { let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); noop_path.push("tests"); noop_path.push("fixtures"); @@ -2138,6 +2175,7 @@ fn cli_program_deploy_with_args(compute_unit_price: Option) { skip_fee_check: false, compute_unit_price, max_sign_attempts: 5, + use_rpc, }); config.output_format = OutputFormat::JsonCompact; let response = process_command(&config); @@ -2238,9 +2276,3 @@ fn cli_program_deploy_with_args(compute_unit_price: Option) { ); } } - -#[test] -fn test_cli_program_deploy_with_compute_unit_price() { - cli_program_deploy_with_args(Some(1000)); - cli_program_deploy_with_args(None); -} diff --git a/client/src/connection_cache.rs b/client/src/connection_cache.rs index 216687aecf916c..1b00f13122d079 100644 --- a/client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -229,7 +229,8 @@ mod tests { crossbeam_channel::unbounded, solana_sdk::{net::DEFAULT_TPU_COALESCE, signature::Keypair}, solana_streamer::{ - nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, quic::SpawnServerResult, + nonblocking::quic::{DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, + quic::SpawnServerResult, streamer::StakedNodes, }, std::{ @@ -273,6 +274,7 @@ mod tests { staked_nodes, 10, 10, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 158614b32d7963..bedddd4c5604d7 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -1229,7 +1229,6 @@ mod tests { bank.clone(), None, bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/core/src/banking_stage/consume_worker.rs b/core/src/banking_stage/consume_worker.rs index d3a53aa42e91b8..404554a48eea14 100644 --- a/core/src/banking_stage/consume_worker.rs +++ b/core/src/banking_stage/consume_worker.rs @@ -672,7 +672,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::new_unique(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 093ee533aa45fb..667ffa8c7c6066 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -869,7 +869,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::new_unique(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -976,7 +975,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &solana_sdk::pubkey::new_rand(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1045,7 +1043,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1190,7 +1187,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1332,7 +1328,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1422,7 +1417,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1576,7 +1570,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1776,7 +1769,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &solana_sdk::pubkey::new_rand(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1877,7 +1869,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &pubkey, blockstore.clone(), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -2022,7 +2013,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::new_unique(), blockstore.clone(), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/core/src/banking_stage/forward_worker.rs b/core/src/banking_stage/forward_worker.rs index 255f1b8e01be99..a0be381086878b 100644 --- a/core/src/banking_stage/forward_worker.rs +++ b/core/src/banking_stage/forward_worker.rs @@ -139,7 +139,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::new_unique(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs index 8247c029593ab2..32332501522ffc 100644 --- a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs +++ b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs @@ -687,7 +687,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::new_unique(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/core/src/repair/repair_generic_traversal.rs b/core/src/repair/repair_generic_traversal.rs index f33a9b91e28bd8..d22f1ca0613cef 100644 --- a/core/src/repair/repair_generic_traversal.rs +++ b/core/src/repair/repair_generic_traversal.rs @@ -270,7 +270,7 @@ pub mod test { &mut processed_slots, 1, ); - assert_eq!(repairs, [ShredRepairType::Shred(1, 4)]); + assert_eq!(repairs, [ShredRepairType::Shred(1, 30)]); } fn add_tree_with_missing_shreds( diff --git a/core/src/repair/serve_repair.rs b/core/src/repair/serve_repair.rs index a4c676bf760eb6..67163a45e089e9 100644 --- a/core/src/repair/serve_repair.rs +++ b/core/src/repair/serve_repair.rs @@ -2163,7 +2163,7 @@ mod tests { // TODO: The test previously relied on corrupting shred payload // size which we no longer want to expose. Current test no longer // covers packet size check in repair_response_packet_from_bytes. - shreds.remove(0); + shreds.retain(|shred| shred.slot() != 1); blockstore .insert_shreds(shreds, None, false) .expect("Expect successful ledger write"); @@ -2192,7 +2192,7 @@ mod tests { let expected = vec![repair_response::repair_response_packet( &blockstore, 2, - 0, + 31, // shred_index &socketaddr_any!(), nonce, ) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 121b11989b27ab..8b4e433ef4bc25 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1942,16 +1942,17 @@ impl ReplayStage { assert!(!poh_recorder.read().unwrap().has_bank()); - let (poh_slot, parent_slot) = match poh_recorder.read().unwrap().reached_leader_slot() { - PohLeaderStatus::Reached { - poh_slot, - parent_slot, - } => (poh_slot, parent_slot), - PohLeaderStatus::NotReached => { - trace!("{} poh_recorder hasn't reached_leader_slot", my_pubkey); - return; - } - }; + let (poh_slot, parent_slot) = + match poh_recorder.read().unwrap().reached_leader_slot(my_pubkey) { + PohLeaderStatus::Reached { + poh_slot, + parent_slot, + } => (poh_slot, parent_slot), + PohLeaderStatus::NotReached => { + trace!("{} poh_recorder hasn't reached_leader_slot", my_pubkey); + return; + } + }; trace!("{} reached_leader_slot", my_pubkey); @@ -4380,7 +4381,6 @@ pub(crate) mod tests { working_bank.clone(), None, working_bank.ticks_per_slot(), - &Pubkey::default(), blockstore.clone(), &leader_schedule_cache, &PohConfig::default(), diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 90ba3ea8a73b46..0c7dccf24a3537 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -104,14 +104,6 @@ impl ShredFetchStage { // Limit shreds to 2 epochs away. let max_slot = last_slot + 2 * slots_per_epoch; - let should_drop_legacy_shreds = |shred_slot| { - check_feature_activation( - &feature_set::drop_legacy_shreds::id(), - shred_slot, - &feature_set, - &epoch_schedule, - ) - }; let enable_chained_merkle_shreds = |shred_slot| { check_feature_activation( &feature_set::enable_chained_merkle_shreds::id(), @@ -128,7 +120,6 @@ impl ShredFetchStage { last_root, max_slot, shred_version, - should_drop_legacy_shreds, enable_chained_merkle_shreds, &mut stats, ) @@ -427,170 +418,3 @@ fn check_feature_activation( } } } - -#[cfg(test)] -mod tests { - use { - super::*, - solana_ledger::{ - blockstore::MAX_DATA_SHREDS_PER_SLOT, - shred::{ReedSolomonCache, Shred, ShredFlags}, - }, - solana_sdk::packet::Packet, - }; - - #[test] - fn test_data_code_same_index() { - solana_logger::setup(); - let mut packet = Packet::default(); - let mut stats = ShredFetchStats::default(); - - let slot = 2; - let shred_version = 45189; - let shred = Shred::new_from_data( - slot, - 3, // shred index - 1, // parent offset - &[], // data - ShredFlags::LAST_SHRED_IN_SLOT, - 0, // reference_tick - shred_version, - 3, // fec_set_index - ); - shred.copy_to_packet(&mut packet); - - let last_root = 0; - let last_slot = 100; - let slots_per_epoch = 10; - let max_slot = last_slot + 2 * slots_per_epoch; - assert!(!should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - let coding = solana_ledger::shred::Shredder::generate_coding_shreds( - &[shred], - 3, // next_code_index - &ReedSolomonCache::default(), - ); - coding[0].copy_to_packet(&mut packet); - assert!(!should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - } - - #[test] - fn test_shred_filter() { - solana_logger::setup(); - let mut packet = Packet::default(); - let mut stats = ShredFetchStats::default(); - let last_root = 0; - let last_slot = 100; - let slots_per_epoch = 10; - let shred_version = 59445; - let max_slot = last_slot + 2 * slots_per_epoch; - - // packet size is 0, so cannot get index - assert!(should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - assert_eq!(stats.index_overrun, 1); - let shred = Shred::new_from_data( - 2, // slot - 3, // index - 1, // parent_offset - &[], // data - ShredFlags::LAST_SHRED_IN_SLOT, - 0, // reference_tick - shred_version, - 0, // fec_set_index - ); - shred.copy_to_packet(&mut packet); - - // rejected slot is 2, root is 3 - assert!(should_discard_shred( - &packet, - 3, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - assert_eq!(stats.slot_out_of_range, 1); - - assert!(should_discard_shred( - &packet, - last_root, - max_slot, - 345, // shred_version - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - assert_eq!(stats.shred_version_mismatch, 1); - - // Accepted for 1,3 - assert!(!should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - - let shred = Shred::new_from_data( - 1_000_000, - 3, - 0, - &[], - ShredFlags::LAST_SHRED_IN_SLOT, - 0, - 0, - 0, - ); - shred.copy_to_packet(&mut packet); - - // Slot 1 million is too high - assert!(should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - - let index = MAX_DATA_SHREDS_PER_SLOT as u32; - let shred = Shred::new_from_data(5, index, 0, &[], ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 0); - shred.copy_to_packet(&mut packet); - assert!(should_discard_shred( - &packet, - last_root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats, - )); - } -} diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 0456a33a8d91f4..a37e28cb571815 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -33,7 +33,7 @@ use { solana_runtime::{bank_forks::BankForks, prioritization_fee_cache::PrioritizationFeeCache}, solana_sdk::{clock::Slot, pubkey::Pubkey, quic::NotifyKeyUpdate, signature::Keypair}, solana_streamer::{ - nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, + nonblocking::quic::{DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, quic::{spawn_server, SpawnServerResult, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, streamer::StakedNodes, }, @@ -167,6 +167,7 @@ impl Tpu { staked_nodes.clone(), MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, tpu_coalesce, ) @@ -191,6 +192,7 @@ impl Tpu { staked_nodes.clone(), MAX_STAKED_CONNECTIONS.saturating_add(MAX_UNSTAKED_CONNECTIONS), 0, // Prevent unstaked nodes from forwarding transactions + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, tpu_coalesce, ) diff --git a/core/src/tvu.rs b/core/src/tvu.rs index d498ab405d39aa..feaf0a9834d17c 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -135,7 +135,7 @@ impl Tvu { wait_to_vote_slot: Option, accounts_background_request_sender: AbsRequestSender, log_messages_bytes_limit: Option, - connection_cache: &Arc, + connection_cache: Option<&Arc>, prioritization_fee_cache: &Arc, banking_tracer: Arc, turbine_quic_endpoint_sender: AsyncSender<(SocketAddr, Bytes)>, @@ -276,16 +276,19 @@ impl Tvu { tower_storage, ); - let warm_quic_cache_service = if connection_cache.use_quic() { - Some(WarmQuicCacheService::new( - connection_cache.clone(), - cluster_info.clone(), - poh_recorder.clone(), - exit.clone(), - )) - } else { - None - }; + let warm_quic_cache_service = connection_cache.and_then(|connection_cache| { + if connection_cache.use_quic() { + Some(WarmQuicCacheService::new( + connection_cache.clone(), + cluster_info.clone(), + poh_recorder.clone(), + exit.clone(), + )) + } else { + None + } + }); + let (cost_update_sender, cost_update_receiver) = unbounded(); let cost_update_service = CostUpdateService::new(blockstore.clone(), cost_update_receiver); @@ -498,7 +501,7 @@ pub mod tests { None, AbsRequestSender::default(), None, - &Arc::new(ConnectionCache::new("connection_cache_test")), + Some(&Arc::new(ConnectionCache::new("connection_cache_test"))), &ignored_prioritization_fee_cache, BankingTracer::new_disabled(), turbine_quic_endpoint_sender, diff --git a/core/src/validator.rs b/core/src/validator.rs index 3fd24d10177c82..6d2a9fb47af9ca 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -925,7 +925,6 @@ impl Validator { bank.clone(), None, bank.ticks_per_slot(), - &id, blockstore.clone(), blockstore.get_new_shred_signal(0), &leader_schedule_cache, @@ -1307,7 +1306,7 @@ impl Validator { config.wait_to_vote_slot, accounts_background_request_sender, config.runtime_config.log_messages_bytes_limit, - &connection_cache, + json_rpc_service.is_some().then_some(&connection_cache), // for the cache warmer only used for STS for RPC service &prioritization_fee_cache, banking_tracer.clone(), turbine_quic_endpoint_sender.clone(), diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 504776db1e1a25..fdbc1894e804bd 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -169,6 +169,11 @@ fn run_check_duplicate( shred_slot, &root_bank, ); + let chained_merkle_conflict_duplicate_proofs = cluster_nodes::check_feature_activation( + &feature_set::chained_merkle_conflict_duplicate_proofs::id(), + shred_slot, + &root_bank, + ); let (shred1, shred2) = match shred { PossibleDuplicateShred::LastIndexConflict(shred, conflict) | PossibleDuplicateShred::ErasureConflict(shred, conflict) => { @@ -196,6 +201,24 @@ fn run_check_duplicate( return Ok(()); } } + PossibleDuplicateShred::ChainedMerkleRootConflict(shred, conflict) => { + if chained_merkle_conflict_duplicate_proofs { + // Although this proof can be immediately stored on detection, we wait until + // here in order to check the feature flag, as storage in blockstore can + // preclude the detection of other duplicate proofs in this slot + if blockstore.has_duplicate_shreds_in_slot(shred_slot) { + return Ok(()); + } + blockstore.store_duplicate_slot( + shred_slot, + conflict.clone(), + shred.clone().into_payload(), + )?; + (shred, conflict) + } else { + return Ok(()); + } + } PossibleDuplicateShred::Exists(shred) => { // Unlike the other cases we have to wait until here to decide to handle the duplicate and store // in blockstore. This is because the duplicate could have been part of the same insert batch, diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index 56b75010f93fe9..ec3440d8527ea8 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -1931,14 +1931,17 @@ impl ClusterInfo { requests .into_par_iter() .with_min_len(1024) - .filter(|(_, _, caller)| match caller.contact_info() { - None => false, - Some(caller) if caller.pubkey() == &self_pubkey => { - warn!("PullRequest ignored, I'm talking to myself"); - self.stats.window_request_loopback.add_relaxed(1); - false + .filter(|(_, _, caller)| match &caller.data { + CrdsData::LegacyContactInfo(_) | CrdsData::ContactInfo(_) => { + if caller.pubkey() == self_pubkey { + warn!("PullRequest ignored, I'm talking to myself"); + self.stats.window_request_loopback.add_relaxed(1); + false + } else { + true + } } - Some(_) => true, + _ => false, }) .map(|(from_addr, filter, caller)| PullData { from_addr, diff --git a/gossip/src/crds_value.rs b/gossip/src/crds_value.rs index ad6422fc2e5188..967cbfa4b9ff18 100644 --- a/gossip/src/crds_value.rs +++ b/gossip/src/crds_value.rs @@ -656,7 +656,7 @@ impl CrdsValue { CrdsData::RestartHeaviestFork(_) => CrdsValueLabel::RestartHeaviestFork(self.pubkey()), } } - pub fn contact_info(&self) -> Option<&LegacyContactInfo> { + pub(crate) fn contact_info(&self) -> Option<&LegacyContactInfo> { match &self.data { CrdsData::LegacyContactInfo(contact_info) => Some(contact_info), _ => None, diff --git a/gossip/src/duplicate_shred.rs b/gossip/src/duplicate_shred.rs index 84c50ea602e8c8..f7ac0153a5c843 100644 --- a/gossip/src/duplicate_shred.rs +++ b/gossip/src/duplicate_shred.rs @@ -595,30 +595,14 @@ pub(crate) mod tests { ), new_rand_data_shred( &mut rng, - next_shred_index + 1, - &shredder, - &leader, - merkle_variant, - false, - ), - ), - ( - new_rand_data_shred( - &mut rng, - next_shred_index + 1, + // With Merkle shreds, last erasure batch is padded with + // empty data shreds. + next_shred_index + if merkle_variant { 30 } else { 1 }, &shredder, &leader, merkle_variant, false, ), - new_rand_data_shred( - &mut rng, - next_shred_index, - &shredder, - &leader, - merkle_variant, - true, - ), ), ( new_rand_data_shred( @@ -638,26 +622,8 @@ pub(crate) mod tests { true, ), ), - ( - new_rand_data_shred( - &mut rng, - next_shred_index, - &shredder, - &leader, - merkle_variant, - true, - ), - new_rand_data_shred( - &mut rng, - next_shred_index + 100, - &shredder, - &leader, - merkle_variant, - true, - ), - ), ]; - for (shred1, shred2) in test_cases.into_iter() { + for (shred1, shred2) in test_cases.iter().flat_map(|(a, b)| [(a, b), (b, a)]) { let chunks: Vec<_> = from_shred( shred1.clone(), Pubkey::new_unique(), // self_pubkey @@ -670,8 +636,8 @@ pub(crate) mod tests { .collect(); assert!(chunks.len() > 4); let (shred3, shred4) = into_shreds(&leader.pubkey(), chunks).unwrap(); - assert_eq!(shred1, shred3); - assert_eq!(shred2, shred4); + assert_eq!(shred1, &shred3); + assert_eq!(shred2, &shred4); } } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5c086d057dae38..55ce4a7ad89e9b 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -67,11 +67,15 @@ use { borrow::Cow, cell::RefCell, cmp, - collections::{hash_map::Entry as HashMapEntry, BTreeSet, HashMap, HashSet, VecDeque}, + collections::{ + btree_map::Entry as BTreeMapEntry, hash_map::Entry as HashMapEntry, BTreeMap, BTreeSet, + HashMap, HashSet, VecDeque, + }, convert::TryInto, fmt::Write, fs, io::{Error as IoError, ErrorKind}, + ops::Bound, path::{Path, PathBuf}, rc::Rc, sync::{ @@ -146,6 +150,7 @@ pub enum PossibleDuplicateShred { LastIndexConflict(/* original */ Shred, /* conflict */ Vec), // The index of this shred conflicts with `slot_meta.last_index` ErasureConflict(/* original */ Shred, /* conflict */ Vec), // The coding shred has a conflict in the erasure_meta MerkleRootConflict(/* original */ Shred, /* conflict */ Vec), // Merkle root conflict in the same fec set + ChainedMerkleRootConflict(/* original */ Shred, /* conflict */ Vec), // Merkle root chaining conflict with previous fec set } impl PossibleDuplicateShred { @@ -155,6 +160,7 @@ impl PossibleDuplicateShred { Self::LastIndexConflict(shred, _) => shred.slot(), Self::ErasureConflict(shred, _) => shred.slot(), Self::MerkleRootConflict(shred, _) => shred.slot(), + Self::ChainedMerkleRootConflict(shred, _) => shred.slot(), } } } @@ -491,6 +497,79 @@ impl Blockstore { self.erasure_meta_cf.get((slot, u64::from(fec_set_index))) } + #[cfg(test)] + fn put_erasure_meta( + &self, + erasure_set: ErasureSetId, + erasure_meta: &ErasureMeta, + ) -> Result<()> { + let (slot, fec_set_index) = erasure_set.store_key(); + self.erasure_meta_cf.put_bytes( + (slot, u64::from(fec_set_index)), + &bincode::serialize(erasure_meta).unwrap(), + ) + } + + /// Attempts to find the previous consecutive erasure set for `erasure_set`. + /// + /// Checks the map `erasure_metas`, if not present scans blockstore. Returns None + /// if the previous consecutive erasure set is not present in either. + fn previous_erasure_set( + &self, + erasure_set: ErasureSetId, + erasure_metas: &BTreeMap>, + ) -> Result> { + let (slot, fec_set_index) = erasure_set.store_key(); + + // Check the previous entry from the in memory map to see if it is the consecutive + // set to `erasure set` + let candidate_erasure_entry = erasure_metas + .range(( + Bound::Included(ErasureSetId::new(slot, 0)), + Bound::Excluded(erasure_set), + )) + .next_back(); + let candidate_erasure_set = candidate_erasure_entry + .filter(|(_, candidate_erasure_meta)| { + candidate_erasure_meta.as_ref().next_fec_set_index() == Some(fec_set_index) + }) + .map(|(candidate_erasure_set, _)| *candidate_erasure_set); + if candidate_erasure_set.is_some() { + return Ok(candidate_erasure_set); + } + + // Consecutive set was not found in memory, scan blockstore for a potential candidate + let Some(((_, candidate_fec_set_index), candidate_erasure_meta)) = self + .erasure_meta_cf + .iter(IteratorMode::From( + (slot, u64::from(fec_set_index)), + IteratorDirection::Reverse, + ))? + // `find` here, to skip the first element in case the erasure meta for fec_set_index is already present + .find(|((_, candidate_fec_set_index), _)| { + *candidate_fec_set_index != u64::from(fec_set_index) + }) + // Do not consider sets from the previous slot + .filter(|((candidate_slot, _), _)| *candidate_slot == slot) + else { + // No potential candidates + return Ok(None); + }; + let candidate_fec_set_index = u32::try_from(candidate_fec_set_index) + .expect("fec_set_index from a previously inserted shred should fit in u32"); + let candidate_erasure_set = ErasureSetId::new(slot, candidate_fec_set_index); + let candidate_erasure_meta: ErasureMeta = deserialize(candidate_erasure_meta.as_ref())?; + + // Check if this is actually the consecutive erasure set + let Some(next_fec_set_index) = candidate_erasure_meta.next_fec_set_index() else { + return Err(BlockstoreError::InvalidErasureConfig); + }; + if next_fec_set_index == fec_set_index { + return Ok(Some(candidate_erasure_set)); + } + Ok(None) + } + fn merkle_root_meta(&self, erasure_set: ErasureSetId) -> Result> { self.merkle_root_meta_cf.get(erasure_set.store_key()) } @@ -763,7 +842,7 @@ impl Blockstore { fn try_shred_recovery( &self, - erasure_metas: &HashMap>, + erasure_metas: &BTreeMap>, index_working_set: &mut HashMap, prev_inserted_shreds: &HashMap, reed_solomon_cache: &ReedSolomonCache, @@ -882,7 +961,7 @@ impl Blockstore { let mut write_batch = self.db.batch()?; let mut just_inserted_shreds = HashMap::with_capacity(shreds.len()); - let mut erasure_metas = HashMap::new(); + let mut erasure_metas = BTreeMap::new(); let mut merkle_root_metas = HashMap::new(); let mut slot_meta_working_set = HashMap::new(); let mut index_working_set = HashMap::new(); @@ -1050,6 +1129,66 @@ impl Blockstore { &mut write_batch, )?; + for (erasure_set, working_erasure_meta) in erasure_metas.iter() { + if !working_erasure_meta.should_write() { + // Not a new erasure meta + continue; + } + let (slot, _) = erasure_set.store_key(); + if self.has_duplicate_shreds_in_slot(slot) { + continue; + } + // First coding shred from this erasure batch, check the forward merkle root chaining + let erasure_meta = working_erasure_meta.as_ref(); + let shred_id = ShredId::new( + slot, + erasure_meta + .first_received_coding_shred_index() + .expect("First received coding index must exist for all erasure metas"), + ShredType::Code, + ); + let shred = just_inserted_shreds + .get(&shred_id) + .expect("Erasure meta was just created, initial shred must exist"); + + self.check_forward_chained_merkle_root_consistency( + shred, + erasure_meta, + &just_inserted_shreds, + &mut merkle_root_metas, + &mut duplicate_shreds, + ); + } + + for (erasure_set, working_merkle_root_meta) in merkle_root_metas.iter() { + if !working_merkle_root_meta.should_write() { + // Not a new merkle root meta + continue; + } + let (slot, _) = erasure_set.store_key(); + if self.has_duplicate_shreds_in_slot(slot) { + continue; + } + // First shred from this erasure batch, check the backwards merkle root chaining + let merkle_root_meta = working_merkle_root_meta.as_ref(); + let shred_id = ShredId::new( + slot, + merkle_root_meta.first_received_shred_index(), + merkle_root_meta.first_received_shred_type(), + ); + let shred = just_inserted_shreds + .get(&shred_id) + .expect("Merkle root meta was just created, initial shred must exist"); + + self.check_backwards_chained_merkle_root_consistency( + shred, + &just_inserted_shreds, + &erasure_metas, + &merkle_root_metas, + &mut duplicate_shreds, + ); + } + for (erasure_set, working_erasure_meta) in erasure_metas { if !working_erasure_meta.should_write() { // No need to rewrite the column @@ -1232,11 +1371,31 @@ impl Blockstore { Ok(insert_results.completed_data_set_infos) } + #[cfg(test)] + fn insert_shred_return_duplicate( + &self, + shred: Shred, + leader_schedule: &LeaderScheduleCache, + ) -> Vec { + let insert_results = self + .do_insert_shreds( + vec![shred], + vec![false], + Some(leader_schedule), + false, + None, // retransmit-sender + &ReedSolomonCache::default(), + &mut BlockstoreInsertionMetrics::default(), + ) + .unwrap(); + insert_results.duplicate_shreds + } + #[allow(clippy::too_many_arguments)] fn check_insert_coding_shred( &self, shred: Shred, - erasure_metas: &mut HashMap>, + erasure_metas: &mut BTreeMap>, merkle_root_metas: &mut HashMap>, index_working_set: &mut HashMap, write_batch: &mut WriteBatch, @@ -1444,7 +1603,7 @@ impl Blockstore { fn check_insert_data_shred( &self, shred: Shred, - erasure_metas: &mut HashMap>, + erasure_metas: &mut BTreeMap>, merkle_root_metas: &mut HashMap>, index_working_set: &mut HashMap, slot_meta_working_set: &mut HashMap, @@ -1539,7 +1698,7 @@ impl Blockstore { just_inserted_shreds.insert(shred.id(), shred); index_meta_working_set_entry.did_insert_occur = true; slot_meta_entry.did_insert_occur = true; - if let HashMapEntry::Vacant(entry) = erasure_metas.entry(erasure_set) { + if let BTreeMapEntry::Vacant(entry) = erasure_metas.entry(erasure_set) { if let Some(meta) = self.erasure_meta(erasure_set).unwrap() { entry.insert(WorkingEntry::Clean(meta)); } @@ -1655,6 +1814,175 @@ impl Blockstore { false } + /// Returns true if there is no chaining conflict between + /// the `shred` and `merkle_root_meta` of the next FEC set, + /// or if shreds from the next set are yet to be received. + /// + /// Otherwise return false and add duplicate proof to + /// `duplicate_shreds`. + /// + /// This is intended to be used right after `shred`'s `erasure_meta` + /// has been created for the first time. + fn check_forward_chained_merkle_root_consistency( + &self, + shred: &Shred, + erasure_meta: &ErasureMeta, + just_inserted_shreds: &HashMap, + merkle_root_metas: &mut HashMap>, + duplicate_shreds: &mut Vec, + ) -> bool { + debug_assert!(erasure_meta.check_coding_shred(shred)); + let slot = shred.slot(); + let erasure_set = shred.erasure_set(); + + // If a shred from the next fec set has already been inserted, check the chaining + let Some(next_fec_set_index) = erasure_meta.next_fec_set_index() else { + error!("Invalid erasure meta, unable to compute next fec set index {erasure_meta:?}"); + return false; + }; + let next_erasure_set = ErasureSetId::new(slot, next_fec_set_index); + let Some(next_merkle_root_meta) = merkle_root_metas + .get(&next_erasure_set) + .map(WorkingEntry::as_ref) + .map(Cow::Borrowed) + .or_else(|| { + self.merkle_root_meta(next_erasure_set) + .unwrap() + .map(Cow::Owned) + }) + else { + // No shred from the next fec set has been received + return true; + }; + let next_shred_id = ShredId::new( + slot, + next_merkle_root_meta.first_received_shred_index(), + next_merkle_root_meta.first_received_shred_type(), + ); + let next_shred = + Self::get_shred_from_just_inserted_or_db(self, just_inserted_shreds, next_shred_id) + .expect("Shred indicated by merkle root meta must exist") + .into_owned(); + let merkle_root = shred.merkle_root().ok(); + let chained_merkle_root = shred::layout::get_chained_merkle_root(&next_shred); + + if !self.check_chaining(merkle_root, chained_merkle_root) { + warn!( + "Received conflicting chained merkle roots for slot: {slot}, + shred {erasure_set:?} type {:?} has merkle root {merkle_root:?}, however + next fec set shred {next_erasure_set:?} type {:?} chains to merkle root {chained_merkle_root:?}. + Reporting as duplicate", + shred.shred_type(), + next_merkle_root_meta.first_received_shred_type(), + ); + + if !self.has_duplicate_shreds_in_slot(shred.slot()) { + duplicate_shreds.push(PossibleDuplicateShred::ChainedMerkleRootConflict( + shred.clone(), + next_shred, + )); + } + return false; + } + + true + } + + /// Returns true if there is no chaining conflict between + /// the `shred` and `merkle_root_meta` of the previous FEC set, + /// or if shreds from the previous set are yet to be received. + /// + /// Otherwise return false and add duplicate proof to + /// `duplicate_shreds`. + /// + /// This is intended to be used right after `shred`'s `merkle_root_meta` + /// has been created for the first time. + fn check_backwards_chained_merkle_root_consistency( + &self, + shred: &Shred, + just_inserted_shreds: &HashMap, + erasure_metas: &BTreeMap>, + merkle_root_metas: &HashMap>, + duplicate_shreds: &mut Vec, + ) -> bool { + let slot = shred.slot(); + let erasure_set = shred.erasure_set(); + let fec_set_index = shred.fec_set_index(); + + if fec_set_index == 0 { + // Although the first fec set chains to the last fec set of the parent block, + // if this chain is incorrect we do not know which block is the duplicate until votes + // are received. We instead delay this check until the block reaches duplicate + // confirmation. + return true; + } + + // If a shred from the previous fec set has already been inserted, check the chaining. + // Since we cannot compute the previous fec set index, we check the in memory map, otherwise + // check the previous key from blockstore to see if it is consecutive with our current set. + let Some(prev_erasure_set) = self + .previous_erasure_set(erasure_set, erasure_metas) + .expect("Expect database operations to succeed") + else { + // No shreds from the previous erasure batch have been received, + // so nothing to check. Once the previous erasure batch is received, + // we will verify this chain through the forward check above. + return true; + }; + + let prev_merkle_root_meta = merkle_root_metas + .get(&prev_erasure_set) + .map(WorkingEntry::as_ref) + .map(Cow::Borrowed) + .or_else(|| { + self.merkle_root_meta(prev_erasure_set) + .unwrap() + .map(Cow::Owned) + }) + .expect("merkle root meta must exist for erasure meta"); + let prev_shred_id = ShredId::new( + slot, + prev_merkle_root_meta.first_received_shred_index(), + prev_merkle_root_meta.first_received_shred_type(), + ); + let prev_shred = + Self::get_shred_from_just_inserted_or_db(self, just_inserted_shreds, prev_shred_id) + .expect("Shred indicated by merkle root meta must exist") + .into_owned(); + let merkle_root = shred::layout::get_merkle_root(&prev_shred); + let chained_merkle_root = shred.chained_merkle_root().ok(); + + if !self.check_chaining(merkle_root, chained_merkle_root) { + warn!( + "Received conflicting chained merkle roots for slot: {slot}, + shred {:?} type {:?} chains to merkle root {chained_merkle_root:?}, however + previous fec set shred {prev_erasure_set:?} type {:?} has merkle root {merkle_root:?}. + Reporting as duplicate", + shred.erasure_set(), + shred.shred_type(), + prev_merkle_root_meta.first_received_shred_type(), + ); + + if !self.has_duplicate_shreds_in_slot(shred.slot()) { + duplicate_shreds.push(PossibleDuplicateShred::ChainedMerkleRootConflict( + shred.clone(), + prev_shred, + )); + } + return false; + } + + true + } + + /// Checks if the chained merkle root == merkle root + /// + /// Returns true if no conflict, or if chained merkle roots are not enabled + fn check_chaining(&self, merkle_root: Option, chained_merkle_root: Option) -> bool { + chained_merkle_root.is_none() // Chained merkle roots have not been enabled yet + || chained_merkle_root == merkle_root + } + fn should_insert_data_shred( &self, shred: &Shred, @@ -6939,7 +7267,7 @@ pub mod tests { let (_, coding_shreds, _) = setup_erasure_shreds(slot, parent_slot, 10); let coding_shred = coding_shreds[index as usize].clone(); - let mut erasure_metas = HashMap::new(); + let mut erasure_metas = BTreeMap::new(); let mut merkle_root_metas = HashMap::new(); let mut index_working_set = HashMap::new(); let mut just_received_shreds = HashMap::new(); @@ -7134,7 +7462,7 @@ pub mod tests { setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); let data_shred = data_shreds[0].clone(); - let mut erasure_metas = HashMap::new(); + let mut erasure_metas = BTreeMap::new(); let mut merkle_root_metas = HashMap::new(); let mut index_working_set = HashMap::new(); let mut just_received_shreds = HashMap::new(); @@ -7351,7 +7679,7 @@ pub mod tests { 0, // version ); - let mut erasure_metas = HashMap::new(); + let mut erasure_metas = BTreeMap::new(); let mut merkle_root_metas = HashMap::new(); let mut index_working_set = HashMap::new(); let mut just_received_shreds = HashMap::new(); @@ -7464,7 +7792,7 @@ pub mod tests { assert_eq!(slot_meta.last_index, Some(num_shreds - 1)); assert!(slot_meta.is_full()); - let (shreds, _) = make_slot_entries(0, 0, 22, /*merkle_variant:*/ true); + let (shreds, _) = make_slot_entries(0, 0, 600, /*merkle_variant:*/ true); assert!(shreds.len() > num_shreds as usize); blockstore.insert_shreds(shreds, None, false).unwrap(); let slot_meta = blockstore.meta(0).unwrap().unwrap(); @@ -9874,6 +10202,22 @@ pub mod tests { parent_slot: u64, num_entries: u64, fec_set_index: u32, + ) -> (Vec, Vec, Arc) { + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + num_entries, + fec_set_index, + Some(Hash::new_from_array(rand::thread_rng().gen())), + ) + } + + fn setup_erasure_shreds_with_index_and_chained_merkle( + slot: u64, + parent_slot: u64, + num_entries: u64, + fec_set_index: u32, + chained_merkle_root: Option, ) -> (Vec, Vec, Arc) { let entries = make_slot_entries_with_transactions(num_entries); let leader_keypair = Arc::new(Keypair::new()); @@ -9882,8 +10226,7 @@ pub mod tests { &leader_keypair, &entries, true, // is_last_in_slot - // chained_merkle_root - Some(Hash::new_from_array(rand::thread_rng().gen())), + chained_merkle_root, fec_set_index, // next_shred_index fec_set_index, // next_code_index true, // merkle_variant @@ -10024,15 +10367,13 @@ pub mod tests { .flat_map(|x| x.0) .collect(); blockstore.insert_shreds(shreds, None, false).unwrap(); - // Should only be one shred in slot 9 - assert!(blockstore - .get_data_shred(unconfirmed_slot, 0) - .unwrap() - .is_some()); - assert!(blockstore - .get_data_shred(unconfirmed_slot, 1) - .unwrap() - .is_none()); + // There are 32 data shreds in slot 9. + for index in 0..32 { + assert_matches!( + blockstore.get_data_shred(unconfirmed_slot, index as u64), + Ok(Some(_)) + ); + } blockstore.set_dead_slot(unconfirmed_slot).unwrap(); // Purge the slot @@ -10065,10 +10406,10 @@ pub mod tests { .into_iter() .flat_map(|x| x.0) .collect(); - assert_eq!(shreds.len(), 2); + assert_eq!(shreds.len(), 2 * 32); - // Save off unconfirmed_slot for later, just one shred at shreds[1] - let unconfirmed_slot_shreds = vec![shreds[1].clone()]; + // Save off unconfirmed_slot for later, just one shred at shreds[32] + let unconfirmed_slot_shreds = vec![shreds[32].clone()]; assert_eq!(unconfirmed_slot_shreds[0].slot(), unconfirmed_slot); // Insert into slot 9 @@ -10795,4 +11136,509 @@ pub mod tests { assert_eq!(read_cost, *cost_table.get(&read_key).unwrap()); } } + + #[test] + fn test_previous_erasure_set() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let mut erasure_metas = BTreeMap::new(); + + let parent_slot = 0; + let prev_slot = 1; + let slot = 2; + let (data_shreds_0, coding_shreds_0, _) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, 0); + let erasure_set_0 = ErasureSetId::new(slot, 0); + let erasure_meta_0 = + ErasureMeta::from_coding_shred(coding_shreds_0.first().unwrap()).unwrap(); + + let prev_fec_set_index = data_shreds_0.len() as u32; + let (data_shreds_prev, coding_shreds_prev, _) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, prev_fec_set_index); + let erasure_set_prev = ErasureSetId::new(slot, prev_fec_set_index); + let erasure_meta_prev = + ErasureMeta::from_coding_shred(coding_shreds_prev.first().unwrap()).unwrap(); + + let (_, coding_shreds_prev_slot, _) = + setup_erasure_shreds_with_index(prev_slot, parent_slot, 10, prev_fec_set_index); + let erasure_set_prev_slot = ErasureSetId::new(prev_slot, prev_fec_set_index); + let erasure_meta_prev_slot = + ErasureMeta::from_coding_shred(coding_shreds_prev_slot.first().unwrap()).unwrap(); + + let fec_set_index = data_shreds_prev.len() as u32 + prev_fec_set_index; + let erasure_set = ErasureSetId::new(slot, fec_set_index); + + // Blockstore is empty + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + None + ); + + // Erasure metas does not contain the previous fec set, but only the one before that + erasure_metas.insert(erasure_set_0, WorkingEntry::Dirty(erasure_meta_0)); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + None + ); + + // Both Erasure metas and blockstore, contain only contain the previous previous fec set + erasure_metas.insert(erasure_set_0, WorkingEntry::Clean(erasure_meta_0)); + blockstore + .put_erasure_meta(erasure_set_0, &erasure_meta_0) + .unwrap(); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + None + ); + + // Erasure meta contains the previous FEC set, blockstore only contains the older + erasure_metas.insert(erasure_set_prev, WorkingEntry::Dirty(erasure_meta_prev)); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + Some(erasure_set_prev) + ); + + // Erasure meta only contains the older, blockstore has the previous fec set + erasure_metas.remove(&erasure_set_prev); + blockstore + .put_erasure_meta(erasure_set_prev, &erasure_meta_prev) + .unwrap(); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + Some(erasure_set_prev) + ); + + // Both contain the previous fec set + erasure_metas.insert(erasure_set_prev, WorkingEntry::Clean(erasure_meta_prev)); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + Some(erasure_set_prev) + ); + + // Works even if the previous fec set has index 0 + assert_eq!( + blockstore + .previous_erasure_set(erasure_set_prev, &erasure_metas) + .unwrap(), + Some(erasure_set_0) + ); + erasure_metas.remove(&erasure_set_0); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set_prev, &erasure_metas) + .unwrap(), + Some(erasure_set_0) + ); + + // Does not cross slot boundary + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + erasure_metas.clear(); + erasure_metas.insert( + erasure_set_prev_slot, + WorkingEntry::Dirty(erasure_meta_prev_slot), + ); + assert_eq!( + erasure_meta_prev_slot.next_fec_set_index().unwrap(), + fec_set_index + ); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + None, + ); + erasure_metas.insert( + erasure_set_prev_slot, + WorkingEntry::Clean(erasure_meta_prev_slot), + ); + blockstore + .put_erasure_meta(erasure_set_prev_slot, &erasure_meta_prev_slot) + .unwrap(); + assert_eq!( + blockstore + .previous_erasure_set(erasure_set, &erasure_metas) + .unwrap(), + None, + ); + } + + #[test] + fn test_chained_merkle_root_consistency_backwards() { + // Insert a coding shred then consistent data and coding shreds from the next FEC set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + data_shreds.len() as u32; + + assert!(blockstore + .insert_shred_return_duplicate(coding_shred.clone(), &leader_schedule,) + .is_empty()); + + let merkle_root = coding_shred.merkle_root().unwrap(); + + // Correctly chained merkle + let (data_shreds, coding_shreds, _) = setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let data_shred = data_shreds[0].clone(); + let coding_shred = coding_shreds[0].clone(); + assert!(blockstore + .insert_shred_return_duplicate(coding_shred, &leader_schedule,) + .is_empty()); + assert!(blockstore + .insert_shred_return_duplicate(data_shred, &leader_schedule,) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_consistency_forwards() { + // Insert a coding shred, then a consistent coding shred from the previous FEC set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + data_shreds.len() as u32; + + // Correctly chained merkle + let merkle_root = coding_shred.merkle_root().unwrap(); + let (_, next_coding_shreds, _) = setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let next_coding_shred = next_coding_shreds[0].clone(); + + assert!(blockstore + .insert_shred_return_duplicate(next_coding_shred, &leader_schedule,) + .is_empty()); + + // Insert previous FEC set + assert!(blockstore + .insert_shred_return_duplicate(coding_shred, &leader_schedule,) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_across_slots_backwards() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, _, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let data_shred = data_shreds[0].clone(); + + assert!(blockstore + .insert_shred_return_duplicate(data_shred.clone(), &leader_schedule,) + .is_empty()); + + // Incorrectly chained merkle for next slot + let merkle_root = Hash::new_unique(); + assert!(merkle_root != data_shred.merkle_root().unwrap()); + let (next_slot_data_shreds, next_slot_coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot + 1, + slot, + 10, + fec_set_index, + Some(merkle_root), + ); + let next_slot_data_shred = next_slot_data_shreds[0].clone(); + let next_slot_coding_shred = next_slot_coding_shreds[0].clone(); + assert!(blockstore + .insert_shred_return_duplicate(next_slot_coding_shred, &leader_schedule,) + .is_empty()); + assert!(blockstore + .insert_shred_return_duplicate(next_slot_data_shred, &leader_schedule) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_across_slots_forwards() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (_, coding_shreds, _) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred = coding_shreds[0].clone(); + + // Incorrectly chained merkle for next slot + let merkle_root = Hash::new_unique(); + assert!(merkle_root != coding_shred.merkle_root().unwrap()); + let (next_slot_data_shreds, _, leader_schedule) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot + 1, + slot, + 10, + fec_set_index, + Some(merkle_root), + ); + let next_slot_data_shred = next_slot_data_shreds[0].clone(); + + assert!(blockstore + .insert_shred_return_duplicate(next_slot_data_shred.clone(), &leader_schedule,) + .is_empty()); + + // Insert for previous slot + assert!(blockstore + .insert_shred_return_duplicate(coding_shred, &leader_schedule,) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_inconsistency_backwards_insert_code() { + // Insert a coding shred then inconsistent coding shred then inconsistent data shred from the next FEC set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred_previous = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + data_shreds.len() as u32; + + assert!(blockstore + .insert_shred_return_duplicate(coding_shred_previous.clone(), &leader_schedule,) + .is_empty()); + + // Incorrectly chained merkle + let merkle_root = Hash::new_unique(); + assert!(merkle_root != coding_shred_previous.merkle_root().unwrap()); + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let data_shred = data_shreds[0].clone(); + let coding_shred = coding_shreds[0].clone(); + let duplicate_shreds = + blockstore.insert_shred_return_duplicate(coding_shred.clone(), &leader_schedule); + assert_eq!(duplicate_shreds.len(), 1); + assert_eq!( + duplicate_shreds[0], + PossibleDuplicateShred::ChainedMerkleRootConflict( + coding_shred, + coding_shred_previous.into_payload() + ) + ); + + // Should not check again, even though this shred conflicts as well + assert!(blockstore + .insert_shred_return_duplicate(data_shred.clone(), &leader_schedule,) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_inconsistency_backwards_insert_data() { + // Insert a coding shred then inconsistent data shred then inconsistent coding shred from the next FEC set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred_previous = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + data_shreds.len() as u32; + + assert!(blockstore + .insert_shred_return_duplicate(coding_shred_previous.clone(), &leader_schedule,) + .is_empty()); + + // Incorrectly chained merkle + let merkle_root = Hash::new_unique(); + assert!(merkle_root != coding_shred_previous.merkle_root().unwrap()); + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let data_shred = data_shreds[0].clone(); + let coding_shred = coding_shreds[0].clone(); + + let duplicate_shreds = + blockstore.insert_shred_return_duplicate(data_shred.clone(), &leader_schedule); + assert_eq!(duplicate_shreds.len(), 1); + assert_eq!( + duplicate_shreds[0], + PossibleDuplicateShred::ChainedMerkleRootConflict( + data_shred, + coding_shred_previous.into_payload(), + ) + ); + // Should not check again, even though this shred conflicts as well + assert!(blockstore + .insert_shred_return_duplicate(coding_shred.clone(), &leader_schedule,) + .is_empty()); + } + + #[test] + fn test_chained_merkle_root_inconsistency_forwards() { + // Insert a data shred, then an inconsistent coding shred from the previous FEC set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let fec_set_index = 0; + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, fec_set_index); + let coding_shred = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + data_shreds.len() as u32; + + // Incorrectly chained merkle + let merkle_root = Hash::new_unique(); + assert!(merkle_root != coding_shred.merkle_root().unwrap()); + let (next_data_shreds, _, leader_schedule_next) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let next_data_shred = next_data_shreds[0].clone(); + + assert!(blockstore + .insert_shred_return_duplicate(next_data_shred.clone(), &leader_schedule_next,) + .is_empty()); + + // Insert previous FEC set + let duplicate_shreds = + blockstore.insert_shred_return_duplicate(coding_shred.clone(), &leader_schedule); + + assert_eq!(duplicate_shreds.len(), 1); + assert_eq!( + duplicate_shreds[0], + PossibleDuplicateShred::ChainedMerkleRootConflict( + coding_shred, + next_data_shred.into_payload(), + ) + ); + } + + #[test] + fn test_chained_merkle_root_inconsistency_both() { + // Insert a coding shred from fec_set - 1, and a data shred from fec_set + 10 + // Then insert an inconsistent data shred from fec_set, and finally an + // inconsistent coding shred from fec_set + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let parent_slot = 0; + let slot = 1; + let prev_fec_set_index = 0; + let (prev_data_shreds, prev_coding_shreds, leader_schedule_prev) = + setup_erasure_shreds_with_index(slot, parent_slot, 10, prev_fec_set_index); + let prev_coding_shred = prev_coding_shreds[0].clone(); + let fec_set_index = prev_fec_set_index + prev_data_shreds.len() as u32; + + // Incorrectly chained merkle + let merkle_root = Hash::new_unique(); + assert!(merkle_root != prev_coding_shred.merkle_root().unwrap()); + let (data_shreds, coding_shreds, leader_schedule) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + fec_set_index, + Some(merkle_root), + ); + let data_shred = data_shreds[0].clone(); + let coding_shred = coding_shreds[0].clone(); + let next_fec_set_index = fec_set_index + prev_data_shreds.len() as u32; + + // Incorrectly chained merkle + let merkle_root = Hash::new_unique(); + assert!(merkle_root != data_shred.merkle_root().unwrap()); + let (next_data_shreds, _, leader_schedule_next) = + setup_erasure_shreds_with_index_and_chained_merkle( + slot, + parent_slot, + 10, + next_fec_set_index, + Some(merkle_root), + ); + let next_data_shred = next_data_shreds[0].clone(); + + assert!(blockstore + .insert_shred_return_duplicate(prev_coding_shred.clone(), &leader_schedule_prev,) + .is_empty()); + + assert!(blockstore + .insert_shred_return_duplicate(next_data_shred.clone(), &leader_schedule_next) + .is_empty()); + + // Insert data shred + let duplicate_shreds = + blockstore.insert_shred_return_duplicate(data_shred.clone(), &leader_schedule); + + // Only the backwards check will be performed + assert_eq!(duplicate_shreds.len(), 1); + assert_eq!( + duplicate_shreds[0], + PossibleDuplicateShred::ChainedMerkleRootConflict( + data_shred, + prev_coding_shred.into_payload(), + ) + ); + + // Insert coding shred + let duplicate_shreds = + blockstore.insert_shred_return_duplicate(coding_shred.clone(), &leader_schedule); + + // Now the forwards check will be performed + assert_eq!(duplicate_shreds.len(), 1); + assert_eq!( + duplicate_shreds[0], + PossibleDuplicateShred::ChainedMerkleRootConflict( + coding_shred, + next_data_shred.into_payload(), + ) + ); + } } diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 8b6b44edae61f6..ab7517453584a2 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -151,6 +151,8 @@ pub enum BlockstoreError { MissingTransactionMetadata, #[error("transaction-index overflow")] TransactionIndexOverflow, + #[error("invalid erasure config")] + InvalidErasureConfig, } pub type Result = std::result::Result; diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index c8b5f6cb4fee99..a76d462c610f04 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -125,9 +125,8 @@ pub struct ErasureMeta { set_index: u64, /// First coding index in the FEC set first_coding_index: u64, - /// Size of shards in this erasure set - #[serde(rename = "size")] - __unused_size: usize, + /// Index of the first received coding shred in the FEC set + first_received_coding_index: u64, /// Erasure configuration for this erasure set config: ErasureConfig, } @@ -348,11 +347,12 @@ impl ErasureMeta { num_coding: usize::from(shred.num_coding_shreds().ok()?), }; let first_coding_index = u64::from(shred.first_coding_index()?); + let first_received_coding_index = u64::from(shred.index()); let erasure_meta = ErasureMeta { set_index: u64::from(shred.fec_set_index()), config, first_coding_index, - __unused_size: 0, + first_received_coding_index, }; Some(erasure_meta) } @@ -365,7 +365,7 @@ impl ErasureMeta { let Some(mut other) = Self::from_coding_shred(shred) else { return false; }; - other.__unused_size = self.__unused_size; + other.first_received_coding_index = self.first_received_coding_index; self == &other } @@ -392,6 +392,18 @@ impl ErasureMeta { self.first_coding_index..self.first_coding_index + num_coding } + pub(crate) fn first_received_coding_shred_index(&self) -> Option { + u32::try_from(self.first_received_coding_index).ok() + } + + pub(crate) fn next_fec_set_index(&self) -> Option { + let num_data = u64::try_from(self.config.num_data).ok()?; + self.set_index + .checked_add(num_data) + .map(u32::try_from)? + .ok() + } + pub(crate) fn status(&self, index: &Index) -> ErasureMetaStatus { use ErasureMetaStatus::*; @@ -560,7 +572,7 @@ mod test { set_index, first_coding_index: set_index, config: erasure_config, - __unused_size: 0, + first_received_coding_index: 0, }; let mut rng = thread_rng(); let mut index = Index::new(0); @@ -713,4 +725,54 @@ mod test { assert_eq!(actual, expected); } + + #[test] + fn test_erasure_meta_transition() { + #[derive(Debug, Deserialize, PartialEq, Serialize)] + struct OldErasureMeta { + set_index: u64, + first_coding_index: u64, + #[serde(rename = "size")] + __unused_size: usize, + config: ErasureConfig, + } + + let set_index = 64; + let erasure_config = ErasureConfig { + num_data: 8, + num_coding: 16, + }; + let mut old_erasure_meta = OldErasureMeta { + set_index, + first_coding_index: set_index, + __unused_size: 0, + config: erasure_config, + }; + let mut new_erasure_meta = ErasureMeta { + set_index, + first_coding_index: set_index, + first_received_coding_index: 0, + config: erasure_config, + }; + + assert_eq!( + bincode::serialized_size(&old_erasure_meta).unwrap(), + bincode::serialized_size(&new_erasure_meta).unwrap(), + ); + + assert_eq!( + bincode::deserialize::(&bincode::serialize(&old_erasure_meta).unwrap()) + .unwrap(), + new_erasure_meta + ); + + new_erasure_meta.first_received_coding_index = u64::from(u32::max_value()); + old_erasure_meta.__unused_size = usize::try_from(u32::max_value()).unwrap(); + + assert_eq!( + bincode::deserialize::(&bincode::serialize(&new_erasure_meta).unwrap()) + .unwrap(), + old_erasure_meta + ); + } } diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 2b6f6f136784c2..7ee66eeaf6be9c 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -292,10 +292,14 @@ impl ShredId { } /// Tuple which identifies erasure coding set that the shred belongs to. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub(crate) struct ErasureSetId(Slot, /*fec_set_index:*/ u32); impl ErasureSetId { + pub(crate) fn new(slot: Slot, fec_set_index: u32) -> Self { + Self(slot, fec_set_index) + } + pub(crate) fn slot(&self) -> Slot { self.0 } @@ -310,7 +314,6 @@ impl ErasureSetId { macro_rules! dispatch { ($vis:vis fn $name:ident(&self $(, $arg:ident : $ty:ty)?) $(-> $out:ty)?) => { #[inline] - #[allow(dead_code)] $vis fn $name(&self $(, $arg:$ty)?) $(-> $out)? { match self { Self::ShredCode(shred) => shred.$name($($arg, )?), @@ -728,7 +731,6 @@ pub mod layout { } } - #[allow(dead_code)] pub(crate) fn get_chained_merkle_root(shred: &[u8]) -> Option { let offset = match get_shred_variant(shred).ok()? { ShredVariant::LegacyCode | ShredVariant::LegacyData => None, @@ -1035,7 +1037,6 @@ pub fn should_discard_shred( root: Slot, max_slot: Slot, shred_version: u16, - should_drop_legacy_shreds: impl Fn(Slot) -> bool, enable_chained_merkle_shreds: impl Fn(Slot) -> bool, stats: &mut ShredFetchStats, ) -> bool { @@ -1112,9 +1113,7 @@ pub fn should_discard_shred( } match shred_variant { ShredVariant::LegacyCode | ShredVariant::LegacyData => { - if should_drop_legacy_shreds(slot) { - return true; - } + return true; } ShredVariant::MerkleCode { chained: false, .. } => { stats.num_shreds_merkle_code = stats.num_shreds_merkle_code.saturating_add(1); @@ -1199,7 +1198,9 @@ mod tests { bincode::serialized_size, rand::Rng, rand_chacha::{rand_core::SeedableRng, ChaChaRng}, + rayon::ThreadPoolBuilder, solana_sdk::{shred_version, signature::Signer, signer::keypair::keypair_from_seed}, + std::io::{Cursor, Seek, SeekFrom, Write}, test_case::test_case, }; @@ -1317,185 +1318,271 @@ mod tests { ); } - #[test] - fn test_should_discard_shred() { + #[test_case(false, false)] + #[test_case(false, true)] + #[test_case(true, false)] + #[test_case(true, true)] + fn test_should_discard_shred(chained: bool, is_last_in_slot: bool) { solana_logger::setup(); - let mut packet = Packet::default(); - let root = 1; - let shred_version = 798; - let max_slot = 16; - let shred = Shred::new_from_data( - 2, // slot - 3, // index - 1, // parent_offset - &[], // data - ShredFlags::LAST_SHRED_IN_SLOT, - 0, // reference_tick - shred_version, - 0, // fec_set_index - ); - shred.copy_to_packet(&mut packet); - let mut stats = ShredFetchStats::default(); - assert!(!should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats, ShredFetchStats::default()); - - packet.meta_mut().size = OFFSET_OF_SHRED_VARIANT; - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats.index_overrun, 1); - - packet.meta_mut().size = OFFSET_OF_SHRED_INDEX; - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats.index_overrun, 2); - - packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + 1; - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats.index_overrun, 3); - - packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1; - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats.index_overrun, 4); - - packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + 2; - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(stats.bad_parent_offset, 1); - - let shred = Shred::new_from_parity_shard( - 8, // slot - 2, // index - &[], // parity_shard - 10, // fec_set_index - 30, // num_data - 4, // num_code - 1, // position - shred_version, - ); - shred.copy_to_packet(&mut packet); - assert!(!should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - - let shred = Shred::new_from_data( - 2, // slot - std::u32::MAX - 10, // index - 1, // parent_offset - &[], // data - ShredFlags::LAST_SHRED_IN_SLOT, - 0, // reference_tick - shred_version, - 0, // fec_set_index - ); - shred.copy_to_packet(&mut packet); - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(1, stats.index_out_of_bounds); - - let shred = Shred::new_from_parity_shard( - 8, // slot - 2, // index - &[], // parity_shard - 10, // fec_set_index - 30, // num_data_shreds - 4, // num_coding_shreds - 3, // position - shred_version, - ); - shred.copy_to_packet(&mut packet); - assert!(!should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - packet.buffer_mut()[OFFSET_OF_SHRED_VARIANT] = u8::MAX; - - assert!(should_discard_shred( - &packet, - root, - max_slot, - shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(1, stats.bad_shred_type); - assert_eq!(stats.shred_version_mismatch, 0); - - packet.buffer_mut()[OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + 1] = u8::MAX; - assert!(should_discard_shred( - &packet, - root, - max_slot, + let mut rng = rand::thread_rng(); + let thread_pool = ThreadPoolBuilder::new().num_threads(2).build().unwrap(); + let reed_solomon_cache = ReedSolomonCache::default(); + let keypair = Keypair::new(); + let chained_merkle_root = chained.then(|| Hash::new_from_array(rng.gen())); + let slot = 18_291; + let parent_slot = rng.gen_range(1..slot); + let shred_version = rng.gen(); + let reference_tick = rng.gen_range(1..64); + let next_shred_index = rng.gen_range(0..671); + let next_code_index = rng.gen_range(0..781); + let mut data = vec![0u8; 1200 * 5]; + rng.fill(&mut data[..]); + let shreds = merkle::make_shreds_from_data( + &thread_pool, + &keypair, + chained_merkle_root, + &data[..], + slot, + parent_slot, shred_version, - |_| false, // should_drop_legacy_shreds - |_| true, // enable_chained_merkle_shreds - &mut stats - )); - assert_eq!(1, stats.bad_shred_type); - assert_eq!(stats.shred_version_mismatch, 1); + reference_tick, + is_last_in_slot, + next_shred_index, + next_code_index, + &reed_solomon_cache, + &mut ProcessShredsStats::default(), + ) + .unwrap(); + assert_eq!(shreds.len(), 1); + let shreds: Vec<_> = shreds.into_iter().flatten().map(Shred::from).collect(); + + let root = rng.gen_range(0..parent_slot); + let max_slot = slot + rng.gen_range(1..65536); + let mut packet = Packet::default(); + + // Data shred sanity checks! + { + let shred = shreds.first().unwrap(); + assert_eq!(shred.shred_type(), ShredType::Data); + shred.copy_to_packet(&mut packet); + let mut stats = ShredFetchStats::default(); + assert!(!should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + } + { + let mut packet = packet.clone(); + let mut stats = ShredFetchStats::default(); + packet.meta_mut().size = OFFSET_OF_SHRED_VARIANT; + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_overrun, 1); + + packet.meta_mut().size = OFFSET_OF_SHRED_INDEX; + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_overrun, 2); + + packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + 1; + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_overrun, 3); + + packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1; + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_overrun, 4); + + packet.meta_mut().size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX + 2; + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.bad_parent_offset, 1); + } + { + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version.wrapping_add(1), + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.shred_version_mismatch, 1); + } + { + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + parent_slot + 1, // root + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.slot_out_of_range, 1); + } + { + let parent_offset = 0u16; + { + let mut cursor = Cursor::new(packet.buffer_mut()); + cursor.seek(SeekFrom::Start(83)).unwrap(); + cursor.write_all(&parent_offset.to_le_bytes()).unwrap(); + } + assert_eq!( + layout::get_parent_offset(packet.data(..).unwrap()), + Some(parent_offset) + ); + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.slot_out_of_range, 1); + } + { + let parent_offset = u16::try_from(slot + 1).unwrap(); + { + let mut cursor = Cursor::new(packet.buffer_mut()); + cursor.seek(SeekFrom::Start(83)).unwrap(); + cursor.write_all(&parent_offset.to_le_bytes()).unwrap(); + } + assert_eq!( + layout::get_parent_offset(packet.data(..).unwrap()), + Some(parent_offset) + ); + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.bad_parent_offset, 1); + } + { + let index = std::u32::MAX - 10; + { + let mut cursor = Cursor::new(packet.buffer_mut()); + cursor + .seek(SeekFrom::Start(OFFSET_OF_SHRED_INDEX as u64)) + .unwrap(); + cursor.write_all(&index.to_le_bytes()).unwrap(); + } + assert_eq!(layout::get_index(packet.data(..).unwrap()), Some(index)); + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_out_of_bounds, 1); + } + + // Coding shred sanity checks! + { + let shred = shreds.last().unwrap(); + assert_eq!(shred.shred_type(), ShredType::Code); + shreds.last().unwrap().copy_to_packet(&mut packet); + let mut stats = ShredFetchStats::default(); + assert!(!should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + } + { + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version.wrapping_add(1), + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.shred_version_mismatch, 1); + } + { + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + slot, // root + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.slot_out_of_range, 1); + } + { + let index = u32::try_from(MAX_CODE_SHREDS_PER_SLOT).unwrap(); + { + let mut cursor = Cursor::new(packet.buffer_mut()); + cursor + .seek(SeekFrom::Start(OFFSET_OF_SHRED_INDEX as u64)) + .unwrap(); + cursor.write_all(&index.to_le_bytes()).unwrap(); + } + assert_eq!(layout::get_index(packet.data(..).unwrap()), Some(index)); + let mut stats = ShredFetchStats::default(); + assert!(should_discard_shred( + &packet, + root, + max_slot, + shred_version, + |_| true, // enable_chained_merkle_shreds + &mut stats + )); + assert_eq!(stats.index_out_of_bounds, 1); + } } // Asserts that ShredType is backward compatible with u8. diff --git a/ledger/src/shred/merkle.rs b/ledger/src/shred/merkle.rs index a7cc134824cf31..a6ffb9757ec3df 100644 --- a/ledger/src/shred/merkle.rs +++ b/ledger/src/shred/merkle.rs @@ -127,6 +127,10 @@ impl Shred { dispatch!(fn merkle_root(&self) -> Result); dispatch!(fn proof_size(&self) -> Result); + fn fec_set_index(&self) -> u32 { + self.common_header().fec_set_index + } + fn index(&self) -> u32 { self.common_header().index } @@ -1098,17 +1102,29 @@ pub(super) fn make_shreds_from_data( // If shreds.is_empty() then the data argument was empty. In that case we // want to generate one data shred with empty data. if !data.is_empty() || shreds.is_empty() { + // Should generate at least one data shred (which may have no data). + // Last erasure batch should also be padded with empty data shreds to + // make >= 32 data shreds. This gaurantees that the batch cannot be + // recovered unless 32+ shreds are received from turbine or repair. + let min_num_data_shreds = if is_last_in_slot { + DATA_SHREDS_PER_FEC_BLOCK + } else { + 1 + }; // Find the Merkle proof_size and data_buffer_size // which can embed the remaining data. - let (proof_size, data_buffer_size) = (1u8..32) + let (proof_size, data_buffer_size, num_data_shreds) = (1u8..32) .find_map(|proof_size| { let data_buffer_size = ShredData::capacity(proof_size, chained, resigned).ok()?; let num_data_shreds = (data.len() + data_buffer_size - 1) / data_buffer_size; - let num_data_shreds = num_data_shreds.max(1); + let num_data_shreds = num_data_shreds.max(min_num_data_shreds); let erasure_batch_size = shredder::get_erasure_batch_size(num_data_shreds, is_last_in_slot); - (proof_size == get_proof_size(erasure_batch_size)) - .then_some((proof_size, data_buffer_size)) + (proof_size == get_proof_size(erasure_batch_size)).then_some(( + proof_size, + data_buffer_size, + num_data_shreds, + )) }) .ok_or(Error::UnknownProofSize)?; common_header.shred_variant = ShredVariant::MerkleData { @@ -1117,13 +1133,11 @@ pub(super) fn make_shreds_from_data( resigned, }; common_header.fec_set_index = common_header.index; - let chunks = if data.is_empty() { - // Generate one data shred with empty data. - Either::Left(std::iter::once(data)) - } else { - Either::Right(data.chunks(data_buffer_size)) - }; - for shred in chunks { + for shred in data + .chunks(data_buffer_size) + .chain(std::iter::repeat(&[][..])) + .take(num_data_shreds) + { let shred = new_shred_data(common_header, data_header, shred); shreds.push(shred); common_header.index += 1; @@ -1132,12 +1146,17 @@ pub(super) fn make_shreds_from_data( stats.data_buffer_residual += data_buffer_size - shred.data()?.len(); } } - // Only the very last shred may have residual data buffer. - debug_assert!(shreds.iter().rev().skip(1).all(|shred| { - let proof_size = shred.proof_size().unwrap(); - let capacity = ShredData::capacity(proof_size, chained, resigned).unwrap(); - shred.data().unwrap().len() == capacity - })); + // Only the trailing data shreds may have residual data buffer. + debug_assert!(shreds + .iter() + .rev() + .skip_while(|shred| is_last_in_slot && shred.data().unwrap().is_empty()) + .skip(1) + .all(|shred| { + let proof_size = shred.proof_size().unwrap(); + let capacity = ShredData::capacity(proof_size, chained, resigned).unwrap(); + shred.data().unwrap().len() == capacity + })); // Adjust flags for the very last shred. if let Some(shred) = shreds.last_mut() { shred.data_header.flags |= if is_last_in_slot { @@ -1890,6 +1909,18 @@ mod test { .contains(ShredFlags::LAST_SHRED_IN_SLOT), is_last_in_slot ); + // Assert that the last erasure batch has 32+ data shreds. + if is_last_in_slot { + let fec_set_index = shreds.iter().map(Shred::fec_set_index).max().unwrap(); + assert!( + shreds + .iter() + .filter(|shred| shred.fec_set_index() == fec_set_index) + .filter(|shred| shred.shred_type() == ShredType::Data) + .count() + >= 32 + ) + } // Assert that data shreds can be recovered from coding shreds. let recovered_data_shreds: Vec<_> = shreds .iter() diff --git a/ledger/src/shred/stats.rs b/ledger/src/shred/stats.rs index 60dfa9a79859c2..696d75b3a7b240 100644 --- a/ledger/src/shred/stats.rs +++ b/ledger/src/shred/stats.rs @@ -32,21 +32,21 @@ pub struct ProcessShredsStats { #[derive(Default, Debug, Eq, PartialEq)] pub struct ShredFetchStats { - pub index_overrun: usize, + pub(super) index_overrun: usize, pub shred_count: usize, - pub(crate) num_shreds_merkle_code: usize, - pub(crate) num_shreds_merkle_code_chained: usize, - pub(crate) num_shreds_merkle_data: usize, - pub(crate) num_shreds_merkle_data_chained: usize, + pub(super) num_shreds_merkle_code: usize, + pub(super) num_shreds_merkle_code_chained: usize, + pub(super) num_shreds_merkle_data: usize, + pub(super) num_shreds_merkle_data_chained: usize, pub ping_count: usize, pub ping_err_verify_count: usize, - pub(crate) index_bad_deserialize: usize, - pub(crate) index_out_of_bounds: usize, - pub(crate) slot_bad_deserialize: usize, - pub slot_out_of_range: usize, - pub(crate) bad_shred_type: usize, - pub shred_version_mismatch: usize, - pub(crate) bad_parent_offset: usize, + pub(super) index_bad_deserialize: usize, + pub(super) index_out_of_bounds: usize, + pub(super) slot_bad_deserialize: usize, + pub(super) slot_out_of_range: usize, + pub(super) bad_shred_type: usize, + pub(super) shred_version_mismatch: usize, + pub(super) bad_parent_offset: usize, since: Option, } diff --git a/ledger/src/shredder.rs b/ledger/src/shredder.rs index ba127ef009c7a5..e23ac208a6814a 100644 --- a/ledger/src/shredder.rs +++ b/ledger/src/shredder.rs @@ -513,7 +513,7 @@ mod tests { assert_eq!(verify, shred.verify(pk)); } - fn run_test_data_shredder(slot: Slot, chained: bool) { + fn run_test_data_shredder(slot: Slot, chained: bool, is_last_in_slot: bool) { let keypair = Arc::new(Keypair::new()); // Test that parent cannot be > current slot @@ -538,11 +538,15 @@ mod tests { }) .collect(); - let is_last_in_slot = true; let size = serialized_size(&entries).unwrap() as usize; // Integer division to ensure we have enough shreds to fit all the data let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap(); let num_expected_data_shreds = (size + data_buffer_size - 1) / data_buffer_size; + let num_expected_data_shreds = num_expected_data_shreds.max(if is_last_in_slot { + DATA_SHREDS_PER_FEC_BLOCK + } else { + 1 + }); let num_expected_coding_shreds = get_erasure_batch_size(num_expected_data_shreds, is_last_in_slot) - num_expected_data_shreds; @@ -574,8 +578,8 @@ mod tests { slot, parent_slot, &keypair.pubkey(), - true, - is_last, + true, // verify + is_last && is_last_in_slot, is_last, ); assert!(!data_shred_indexes.contains(&index)); @@ -607,10 +611,12 @@ mod tests { assert_eq!(entries, deshred_entries); } - #[test_case(false)] - #[test_case(true)] - fn test_data_shredder(chained: bool) { - run_test_data_shredder(0x1234_5678_9abc_def0, chained); + #[test_case(false, false)] + #[test_case(false, true)] + #[test_case(true, false)] + #[test_case(true, true)] + fn test_data_shredder(chained: bool, is_last_in_slot: bool) { + run_test_data_shredder(0x1234_5678_9abc_def0, chained, is_last_in_slot); } #[test_case(false)] diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 901292c9dd9d8f..675263f26f6af9 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -3741,6 +3741,7 @@ fn test_kill_partition_switch_threshold_progress() { #[test] #[serial] +#[ignore] #[allow(unused_attributes)] fn test_duplicate_shreds_broadcast_leader() { run_duplicate_shreds_broadcast_leader(true); diff --git a/poh/src/poh_recorder.rs b/poh/src/poh_recorder.rs index 49c2d4dc3d9a88..8cabc193b966b1 100644 --- a/poh/src/poh_recorder.rs +++ b/poh/src/poh_recorder.rs @@ -289,7 +289,6 @@ pub struct PohRecorder { leader_first_tick_height_including_grace_ticks: Option, leader_last_tick_height: u64, // zero if none grace_ticks: u64, - id: Pubkey, blockstore: Arc, leader_schedule_cache: Arc, ticks_per_slot: u64, @@ -314,7 +313,7 @@ impl PohRecorder { if let Some(WorkingBank { bank, start, .. }) = self.working_bank.take() { self.leader_bank_notifier.set_completed(bank.slot()); let next_leader_slot = self.leader_schedule_cache.next_leader_slot( - &self.id, + bank.collector_id(), bank.slot(), &bank, Some(&self.blockstore), @@ -450,18 +449,22 @@ impl PohRecorder { }) } - fn prev_slot_was_mine(&self, current_slot: Slot) -> bool { + fn prev_slot_was_mine(&self, my_pubkey: &Pubkey, current_slot: Slot) -> bool { if let Some(leader_id) = self .leader_schedule_cache .slot_leader_at(current_slot.saturating_sub(1), None) { - leader_id == self.id + &leader_id == my_pubkey } else { false } } - fn reached_leader_tick(&self, leader_first_tick_height_including_grace_ticks: u64) -> bool { + fn reached_leader_tick( + &self, + my_pubkey: &Pubkey, + leader_first_tick_height_including_grace_ticks: u64, + ) -> bool { let target_tick_height = leader_first_tick_height_including_grace_ticks.saturating_sub(1); let ideal_target_tick_height = target_tick_height.saturating_sub(self.grace_ticks); let next_tick_height = self.tick_height.saturating_add(1); @@ -472,7 +475,7 @@ impl PohRecorder { || self.start_tick_height + self.grace_ticks == leader_first_tick_height_including_grace_ticks || (self.tick_height >= ideal_target_tick_height - && (self.prev_slot_was_mine(next_slot) + && (self.prev_slot_was_mine(my_pubkey, next_slot) || !self.is_same_fork_as_previous_leader(next_slot))) } @@ -483,7 +486,7 @@ impl PohRecorder { /// Returns if the leader slot has been reached along with the current poh /// slot and the parent slot (could be a few slots ago if any previous /// leaders needed to be skipped). - pub fn reached_leader_slot(&self) -> PohLeaderStatus { + pub fn reached_leader_slot(&self, my_pubkey: &Pubkey) -> PohLeaderStatus { trace!( "tick_height {}, start_tick_height {}, leader_first_tick_height_including_grace_ticks {:?}, grace_ticks {}, has_bank {}", self.tick_height, @@ -498,7 +501,7 @@ impl PohRecorder { if let Some(leader_first_tick_height_including_grace_ticks) = self.leader_first_tick_height_including_grace_ticks { - if self.reached_leader_tick(leader_first_tick_height_including_grace_ticks) { + if self.reached_leader_tick(my_pubkey, leader_first_tick_height_including_grace_ticks) { assert!(next_tick_height >= self.start_tick_height); let poh_slot = next_poh_slot; let parent_slot = self.start_slot(); @@ -938,7 +941,6 @@ impl PohRecorder { start_bank: Arc, next_leader_slot: Option<(Slot, Slot)>, ticks_per_slot: u64, - id: &Pubkey, blockstore: Arc, clear_bank_signal: Option>, leader_schedule_cache: &Arc, @@ -975,7 +977,6 @@ impl PohRecorder { leader_first_tick_height_including_grace_ticks, leader_last_tick_height, grace_ticks, - id: *id, blockstore, leader_schedule_cache: leader_schedule_cache.clone(), ticks_per_slot, @@ -1009,7 +1010,6 @@ impl PohRecorder { start_bank: Arc, next_leader_slot: Option<(Slot, Slot)>, ticks_per_slot: u64, - id: &Pubkey, blockstore: Arc, leader_schedule_cache: &Arc, poh_config: &PohConfig, @@ -1021,7 +1021,6 @@ impl PohRecorder { start_bank, next_leader_slot, ticks_per_slot, - id, blockstore, None, leader_schedule_cache, @@ -1082,7 +1081,6 @@ pub fn create_test_recorder( bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), blockstore, &leader_schedule_cache, &poh_config, @@ -1133,7 +1131,6 @@ mod tests { bank, Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1160,7 +1157,6 @@ mod tests { bank, Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1186,7 +1182,6 @@ mod tests { bank0.clone(), Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1212,7 +1207,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1239,7 +1233,6 @@ mod tests { bank0.clone(), Some((4, 4)), bank0.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank0)), &PohConfig::default(), @@ -1299,7 +1292,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1345,7 +1337,6 @@ mod tests { bank0.clone(), Some((4, 4)), bank0.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank0)), &PohConfig::default(), @@ -1385,7 +1376,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1424,7 +1414,6 @@ mod tests { bank0.clone(), Some((4, 4)), bank0.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank0)), &PohConfig::default(), @@ -1477,7 +1466,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1514,7 +1502,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1583,7 +1570,6 @@ mod tests { bank0.clone(), Some((4, 4)), bank0.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank0)), &PohConfig::default(), @@ -1635,7 +1621,6 @@ mod tests { bank.clone(), Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1661,7 +1646,6 @@ mod tests { bank.clone(), Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1689,7 +1673,6 @@ mod tests { bank.clone(), Some((4, 4)), DEFAULT_TICKS_PER_SLOT, - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::default()), &PohConfig::default(), @@ -1720,7 +1703,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1748,7 +1730,6 @@ mod tests { bank.clone(), None, bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), Some(sender), &Arc::new(LeaderScheduleCache::default()), @@ -1781,7 +1762,6 @@ mod tests { bank.clone(), Some((4, 4)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -1816,7 +1796,11 @@ mod tests { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()) .expect("Expected to be able to open database ledger"); - let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(2); + let GenesisConfigInfo { + genesis_config, + validator_pubkey, + .. + } = create_genesis_config(2); let bank = Arc::new(Bank::new_for_tests(&genesis_config)); let prev_hash = bank.last_blockhash(); let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank)); @@ -1826,16 +1810,13 @@ mod tests { bank.clone(), None, bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &leader_schedule_cache, &PohConfig::default(), Arc::new(AtomicBool::default()), ); - let bootstrap_validator_id = leader_schedule_cache.slot_leader_at(0, None).unwrap(); - - assert!(poh_recorder.reached_leader_tick(0)); + assert!(poh_recorder.reached_leader_tick(&validator_pubkey, 0)); let grace_ticks = bank.ticks_per_slot() * MAX_GRACE_SLOTS; let new_tick_height = NUM_CONSECUTIVE_LEADER_SLOTS * bank.ticks_per_slot(); @@ -1848,7 +1829,9 @@ mod tests { // False, because the Poh was reset on slot 0, which // is a block produced by the previous leader, so a grace // period must be given - assert!(!poh_recorder.reached_leader_tick(new_tick_height + grace_ticks)); + let test_validator_pubkey = Pubkey::new_unique(); + assert!(!poh_recorder + .reached_leader_tick(&test_validator_pubkey, new_tick_height + grace_ticks)); // Tick `NUM_CONSECUTIVE_LEADER_SLOTS` more times let new_tick_height = 2 * NUM_CONSECUTIVE_LEADER_SLOTS * bank.ticks_per_slot(); @@ -1861,14 +1844,15 @@ mod tests { // none of the previous leader's `NUM_CONSECUTIVE_LEADER_SLOTS` were slots // this Poh built on (previous leader was on different fork). Thus, skip the // grace period. - assert!(poh_recorder.reached_leader_tick(new_tick_height + grace_ticks)); + assert!( + poh_recorder.reached_leader_tick(&test_validator_pubkey, new_tick_height + grace_ticks) + ); // From the bootstrap validator's perspective, it should have reached // the tick because the previous slot was also it's own slot (all slots // belong to the bootstrap leader b/c it's the only staked node!), and // validators don't give grace periods if previous slot was also their own. - poh_recorder.id = bootstrap_validator_id; - assert!(poh_recorder.reached_leader_tick(new_tick_height + grace_ticks)); + assert!(poh_recorder.reached_leader_tick(&validator_pubkey, new_tick_height + grace_ticks)); } #[test] @@ -1878,7 +1862,12 @@ mod tests { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()) .expect("Expected to be able to open database ledger"); - let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(2); + + let GenesisConfigInfo { + genesis_config, + validator_pubkey, + .. + } = create_genesis_config(2); let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); let prev_hash = bank0.last_blockhash(); let (mut poh_recorder, _entry_receiver, _record_receiver) = PohRecorder::new( @@ -1887,7 +1876,6 @@ mod tests { bank0.clone(), None, bank0.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank0)), &PohConfig::default(), @@ -1896,7 +1884,7 @@ mod tests { // Test that with no next leader slot, we don't reach the leader slot assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&validator_pubkey), PohLeaderStatus::NotReached ); @@ -1904,7 +1892,7 @@ mod tests { assert_eq!(bank0.slot(), 0); poh_recorder.reset(bank0.clone(), None); assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&validator_pubkey), PohLeaderStatus::NotReached ); @@ -1933,9 +1921,13 @@ mod tests { .put_meta_bytes(0, &serialize(&parent_meta).unwrap()) .unwrap(); + // Use a key that's different from the previous leader so that grace + // ticks are enforced. + let test_validator_pubkey = Pubkey::new_unique(); + // Test that we don't reach the leader slot because of grace ticks assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::NotReached ); @@ -1944,7 +1936,7 @@ mod tests { assert_eq!(bank1.slot(), 1); poh_recorder.reset(bank1.clone(), Some((2, 2))); assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&validator_pubkey), PohLeaderStatus::Reached { poh_slot: 2, parent_slot: 1, @@ -1962,9 +1954,17 @@ mod tests { // We are not the leader yet, as expected assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::NotReached ); + // Check that if prev slot was mine, grace ticks are ignored + assert_eq!( + poh_recorder.reached_leader_slot(&validator_pubkey), + PohLeaderStatus::Reached { + poh_slot: 3, + parent_slot: 1 + } + ); // Send the grace ticks for _ in 0..bank1.ticks_per_slot() / GRACE_TICKS_FACTOR { @@ -1974,7 +1974,7 @@ mod tests { // We should be the leader now // without sending more ticks, we should be leader now assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::Reached { poh_slot: 3, parent_slot: 1, @@ -1993,7 +1993,7 @@ mod tests { // We are not the leader yet, as expected assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::NotReached ); let bank3 = Arc::new(Bank::new_from_parent(bank2, &Pubkey::default(), 3)); @@ -2002,7 +2002,7 @@ mod tests { // without sending more ticks, we should be leader now assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::Reached { poh_slot: 4, parent_slot: 3, @@ -2023,7 +2023,7 @@ mod tests { // We are overdue to lead assert_eq!( - poh_recorder.reached_leader_slot(), + poh_recorder.reached_leader_slot(&test_validator_pubkey), PohLeaderStatus::Reached { poh_slot: 9, parent_slot: 4, @@ -2045,7 +2045,6 @@ mod tests { bank.clone(), None, bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), @@ -2095,7 +2094,6 @@ mod tests { bank.clone(), Some((2, 2)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/poh/src/poh_service.rs b/poh/src/poh_service.rs index e69db7f119862b..8cd0d40266b37d 100644 --- a/poh/src/poh_service.rs +++ b/poh/src/poh_service.rs @@ -391,9 +391,7 @@ mod tests { solana_measure::measure::Measure, solana_perf::test_tx::test_tx, solana_runtime::bank::Bank, - solana_sdk::{ - clock, hash::hash, pubkey::Pubkey, timing, transaction::VersionedTransaction, - }, + solana_sdk::{clock, hash::hash, timing, transaction::VersionedTransaction}, std::{thread::sleep, time::Duration}, }; @@ -427,7 +425,6 @@ mod tests { bank.clone(), Some((4, 4)), ticks_per_slot, - &Pubkey::default(), blockstore, &leader_schedule_cache, &poh_config, diff --git a/programs/bpf_loader/src/syscalls/mod.rs b/programs/bpf_loader/src/syscalls/mod.rs index 4a166fa1cf9996..6d500528d5e236 100644 --- a/programs/bpf_loader/src/syscalls/mod.rs +++ b/programs/bpf_loader/src/syscalls/mod.rs @@ -1573,14 +1573,24 @@ declare_builtin_function!( } }; + let simplify_alt_bn128_syscall_error_codes = invoke_context + .feature_set + .is_active(&feature_set::simplify_alt_bn128_syscall_error_codes::id()); + let result_point = match calculation(input) { Ok(result_point) => result_point, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; - if result_point.len() != output { + // This can never happen and should be removed when the + // simplify_alt_bn128_syscall_error_codes feature gets activated + if result_point.len() != output && !simplify_alt_bn128_syscall_error_codes { return Ok(AltBn128Error::SliceOutOfBounds.into()); } @@ -1720,10 +1730,19 @@ declare_builtin_function!( ) }) .collect::, Error>>()?; + + let simplify_alt_bn128_syscall_error_codes = invoke_context + .feature_set + .is_active(&feature_set::simplify_alt_bn128_syscall_error_codes::id()); + let hash = match poseidon::hashv(parameters, endianness, inputs.as_slice()) { Ok(hash) => hash, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; hash_result.copy_from_slice(&hash.to_bytes()); @@ -1807,12 +1826,20 @@ declare_builtin_function!( invoke_context.get_check_aligned(), )?; + let simplify_alt_bn128_syscall_error_codes = invoke_context + .feature_set + .is_active(&feature_set::simplify_alt_bn128_syscall_error_codes::id()); + match op { ALT_BN128_G1_COMPRESS => { let result_point = match alt_bn128_g1_compress(input) { Ok(result_point) => result_point, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; call_result.copy_from_slice(&result_point); @@ -1822,7 +1849,11 @@ declare_builtin_function!( let result_point = match alt_bn128_g1_decompress(input) { Ok(result_point) => result_point, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; call_result.copy_from_slice(&result_point); @@ -1832,7 +1863,11 @@ declare_builtin_function!( let result_point = match alt_bn128_g2_compress(input) { Ok(result_point) => result_point, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; call_result.copy_from_slice(&result_point); @@ -1842,7 +1877,11 @@ declare_builtin_function!( let result_point = match alt_bn128_g2_decompress(input) { Ok(result_point) => result_point, Err(e) => { - return Ok(e.into()); + return if simplify_alt_bn128_syscall_error_codes { + Ok(1) + } else { + Ok(e.into()) + }; } }; call_result.copy_from_slice(&result_point); diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 71d5f748f27d94..14f67855c36e86 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -65,7 +65,7 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-sdk", @@ -75,7 +75,7 @@ dependencies = [ [[package]] name = "agave-validator" -version = "1.18.11" +version = "1.18.12" dependencies = [ "agave-geyser-plugin-interface", "chrono", @@ -4238,9 +4238,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring 0.17.3", @@ -4706,7 +4706,7 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "base64 0.21.7", @@ -4729,7 +4729,7 @@ dependencies = [ [[package]] name = "solana-accounts-db" -version = "1.18.11" +version = "1.18.12" dependencies = [ "arrayref", "bincode", @@ -4788,7 +4788,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bytemuck", @@ -4807,7 +4807,7 @@ dependencies = [ [[package]] name = "solana-banks-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "borsh 1.2.1", "futures 0.3.30", @@ -4822,7 +4822,7 @@ dependencies = [ [[package]] name = "solana-banks-interface" -version = "1.18.11" +version = "1.18.12" dependencies = [ "serde", "solana-sdk", @@ -4831,7 +4831,7 @@ dependencies = [ [[package]] name = "solana-banks-server" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "crossbeam-channel", @@ -4849,7 +4849,7 @@ dependencies = [ [[package]] name = "solana-bloom" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bv", "fnv", @@ -4866,7 +4866,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "byteorder 1.5.0", @@ -4883,7 +4883,7 @@ dependencies = [ [[package]] name = "solana-bpf-rust-big-mod-exp" -version = "1.18.11" +version = "1.18.12" dependencies = [ "array-bytes", "serde", @@ -4893,7 +4893,7 @@ dependencies = [ [[package]] name = "solana-bucket-map" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bv", "bytemuck", @@ -4909,7 +4909,7 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "chrono", "clap 2.33.3", @@ -4924,7 +4924,7 @@ dependencies = [ [[package]] name = "solana-cli-config" -version = "1.18.11" +version = "1.18.12" dependencies = [ "dirs-next", "lazy_static", @@ -4938,7 +4938,7 @@ dependencies = [ [[package]] name = "solana-cli-output" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "base64 0.21.7", @@ -4963,7 +4963,7 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -4994,7 +4994,7 @@ dependencies = [ [[package]] name = "solana-compute-budget-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program-runtime", "solana-sdk", @@ -5002,7 +5002,7 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "chrono", @@ -5014,7 +5014,7 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -5034,7 +5034,7 @@ dependencies = [ [[package]] name = "solana-core" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -5109,7 +5109,7 @@ dependencies = [ [[package]] name = "solana-cost-model" -version = "1.18.11" +version = "1.18.12" dependencies = [ "lazy_static", "log", @@ -5131,7 +5131,7 @@ dependencies = [ [[package]] name = "solana-download-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "console", "indicatif", @@ -5143,7 +5143,7 @@ dependencies = [ [[package]] name = "solana-entry" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "crossbeam-channel", @@ -5163,7 +5163,7 @@ dependencies = [ [[package]] name = "solana-faucet" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "byteorder 1.5.0", @@ -5185,7 +5185,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.18.11" +version = "1.18.12" dependencies = [ "block-buffer 0.10.4", "bs58", @@ -5208,7 +5208,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.18.11" +version = "1.18.12" dependencies = [ "proc-macro2", "quote", @@ -5218,7 +5218,7 @@ dependencies = [ [[package]] name = "solana-genesis-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-accounts-db", @@ -5229,7 +5229,7 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-manager" -version = "1.18.11" +version = "1.18.12" dependencies = [ "agave-geyser-plugin-interface", "bs58", @@ -5254,7 +5254,7 @@ dependencies = [ [[package]] name = "solana-gossip" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5302,7 +5302,7 @@ dependencies = [ [[package]] name = "solana-ledger" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "bincode", @@ -5367,7 +5367,7 @@ dependencies = [ [[package]] name = "solana-loader-v4-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-measure", @@ -5378,7 +5378,7 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.18.11" +version = "1.18.12" dependencies = [ "env_logger", "lazy_static", @@ -5387,7 +5387,7 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "solana-sdk", @@ -5395,7 +5395,7 @@ dependencies = [ [[package]] name = "solana-merkle-tree" -version = "1.18.11" +version = "1.18.12" dependencies = [ "fast-math", "solana-program", @@ -5403,7 +5403,7 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "gethostname", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "clap 3.1.6", @@ -5442,7 +5442,7 @@ checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" [[package]] name = "solana-perf" -version = "1.18.11" +version = "1.18.12" dependencies = [ "ahash 0.8.7", "bincode", @@ -5469,7 +5469,7 @@ dependencies = [ [[package]] name = "solana-poh" -version = "1.18.11" +version = "1.18.12" dependencies = [ "core_affinity", "crossbeam-channel", @@ -5485,7 +5485,7 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "ark-bn254", "ark-ec", @@ -5538,7 +5538,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -5564,7 +5564,7 @@ dependencies = [ [[package]] name = "solana-program-test" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "async-trait", @@ -5592,7 +5592,7 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "futures-util", @@ -5615,7 +5615,7 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-mutex", "async-trait", @@ -5640,7 +5640,7 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.18.11" +version = "1.18.12" dependencies = [ "lazy_static", "num_cpus", @@ -5648,7 +5648,7 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.18.11" +version = "1.18.12" dependencies = [ "console", "dialoguer", @@ -5665,7 +5665,7 @@ dependencies = [ [[package]] name = "solana-rpc" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -5720,7 +5720,7 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "base64 0.21.7", @@ -5744,7 +5744,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bs58", @@ -5764,7 +5764,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.18.11" +version = "1.18.12" dependencies = [ "clap 2.33.3", "solana-clap-utils", @@ -5775,7 +5775,7 @@ dependencies = [ [[package]] name = "solana-runtime" -version = "1.18.11" +version = "1.18.12" dependencies = [ "aquamarine", "arrayref", @@ -5850,7 +5850,7 @@ dependencies = [ [[package]] name = "solana-sbf-programs" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "byteorder 1.5.0", @@ -5879,7 +5879,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-128bit" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-sbf-rust-128bit-dep", @@ -5887,21 +5887,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-128bit-dep" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-alloc" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-alt-bn128" -version = "1.18.11" +version = "1.18.12" dependencies = [ "array-bytes", "solana-program", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-alt-bn128-compression" -version = "1.18.11" +version = "1.18.12" dependencies = [ "array-bytes", "solana-program", @@ -5917,21 +5917,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-call-depth" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-caller-access" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-curve25519" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-zk-token-sdk", @@ -5939,14 +5939,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-custom-heap" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-dep-crate" -version = "1.18.11" +version = "1.18.12" dependencies = [ "byteorder 1.5.0", "solana-program", @@ -5954,21 +5954,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-deprecated-loader" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-dup-accounts" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-error-handling" -version = "1.18.11" +version = "1.18.12" dependencies = [ "num-derive 0.3.0", "num-traits", @@ -5978,42 +5978,42 @@ dependencies = [ [[package]] name = "solana-sbf-rust-external-spend" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-finalize" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-get-minimum-delegation" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-inner_instruction_alignment_check" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-instruction-introspection" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke" -version = "1.18.11" +version = "1.18.12" dependencies = [ "rustversion", "solana-program", @@ -6023,49 +6023,49 @@ dependencies = [ [[package]] name = "solana-sbf-rust-invoke-and-error" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke-and-ok" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke-and-return" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoked" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-iter" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-log-data" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-many-args" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-sbf-rust-many-args-dep", @@ -6073,14 +6073,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-many-args-dep" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-mem" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-program-runtime", @@ -6090,7 +6090,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-membuiltins" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-sbf-rust-mem", @@ -6098,21 +6098,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-noop" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-panic" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-param-passing" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-sbf-rust-param-passing-dep", @@ -6120,14 +6120,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-param-passing-dep" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-poseidon" -version = "1.18.11" +version = "1.18.12" dependencies = [ "array-bytes", "solana-program", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-rand" -version = "1.18.11" +version = "1.18.12" dependencies = [ "getrandom 0.2.10", "rand 0.8.5", @@ -6144,14 +6144,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-realloc" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-realloc-invoke" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-sbf-rust-realloc", @@ -6159,7 +6159,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-remaining-compute-units" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-program-runtime", @@ -6169,21 +6169,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-ro-account_modify" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-ro-modify" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sanity" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-program-runtime", @@ -6193,7 +6193,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-secp256k1-recover" -version = "1.18.11" +version = "1.18.12" dependencies = [ "libsecp256k1 0.7.0", "solana-program", @@ -6201,7 +6201,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-sha" -version = "1.18.11" +version = "1.18.12" dependencies = [ "blake3", "solana-program", @@ -6209,21 +6209,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-sibling-instructions" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sibling_inner-instructions" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-simulation" -version = "1.18.11" +version = "1.18.12" dependencies = [ "agave-validator", "solana-logger", @@ -6234,21 +6234,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-spoof1" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-spoof1-system" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sysvar" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", "solana-program-runtime", @@ -6258,21 +6258,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-upgradeable" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-upgraded" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-program", ] [[package]] name = "solana-sdk" -version = "1.18.11" +version = "1.18.12" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6325,7 +6325,7 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bs58", "proc-macro2", @@ -6342,7 +6342,7 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-send-transaction-service" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "log", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "solana-stake-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -6369,7 +6369,7 @@ dependencies = [ [[package]] name = "solana-storage-bigtable" -version = "1.18.11" +version = "1.18.12" dependencies = [ "backoff", "bincode", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bs58", @@ -6416,7 +6416,7 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-channel", "bytes", @@ -6447,7 +6447,7 @@ dependencies = [ [[package]] name = "solana-system-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -6459,7 +6459,7 @@ dependencies = [ [[package]] name = "solana-test-validator" -version = "1.18.11" +version = "1.18.12" dependencies = [ "base64 0.21.7", "bincode", @@ -6489,7 +6489,7 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -6502,7 +6502,7 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "bincode", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.18.11" +version = "1.18.12" dependencies = [ "Inflector", "base64 0.21.7", @@ -6547,7 +6547,7 @@ dependencies = [ [[package]] name = "solana-turbine" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "bytes", @@ -6582,7 +6582,7 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.18.11" +version = "1.18.12" dependencies = [ "async-trait", "solana-connection-cache", @@ -6595,11 +6595,11 @@ dependencies = [ [[package]] name = "solana-unified-scheduler-logic" -version = "1.18.11" +version = "1.18.12" [[package]] name = "solana-unified-scheduler-pool" -version = "1.18.11" +version = "1.18.12" dependencies = [ "solana-ledger", "solana-program-runtime", @@ -6611,7 +6611,7 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "rustc_version", @@ -6625,7 +6625,7 @@ dependencies = [ [[package]] name = "solana-vote" -version = "1.18.11" +version = "1.18.12" dependencies = [ "crossbeam-channel", "itertools", @@ -6642,7 +6642,7 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bincode", "log", @@ -6662,7 +6662,7 @@ dependencies = [ [[package]] name = "solana-wen-restart" -version = "1.18.11" +version = "1.18.12" dependencies = [ "log", "prost", @@ -6681,7 +6681,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program" -version = "1.18.11" +version = "1.18.12" dependencies = [ "bytemuck", "num-derive 0.4.1", @@ -6693,7 +6693,7 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.18.11" +version = "1.18.12" dependencies = [ "aes-gcm-siv", "base64 0.21.7", diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 68dd685e697ef7..c9c6924eeddb8e 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "1.18.11" +version = "1.18.12" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -25,29 +25,29 @@ rand = "0.8" rustversion = "1.0.14" serde = "1.0.112" serde_json = "1.0.56" -solana-account-decoder = { path = "../../account-decoder", version = "=1.18.11" } -solana-accounts-db = { path = "../../accounts-db", version = "=1.18.11" } -solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.18.11" } -solana-cli-output = { path = "../../cli-output", version = "=1.18.11" } -solana-ledger = { path = "../../ledger", version = "=1.18.11" } -solana-logger = { path = "../../logger", version = "=1.18.11" } -solana-measure = { path = "../../measure", version = "=1.18.11" } -solana-program = { path = "../../sdk/program", version = "=1.18.11" } -solana-program-runtime = { path = "../../program-runtime", version = "=1.18.11" } -solana-program-test = { path = "../../program-test", version = "=1.18.11" } -solana-runtime = { path = "../../runtime", version = "=1.18.11" } -solana-sbf-rust-128bit-dep = { path = "rust/128bit_dep", version = "=1.18.11" } -solana-sbf-rust-invoke = { path = "rust/invoke", version = "=1.18.11" } -solana-sbf-rust-invoked = { path = "rust/invoked", version = "=1.18.11", default-features = false } -solana-sbf-rust-many-args-dep = { path = "rust/many_args_dep", version = "=1.18.11" } -solana-sbf-rust-mem = { path = "rust/mem", version = "=1.18.11" } -solana-sbf-rust-param-passing-dep = { path = "rust/param_passing_dep", version = "=1.18.11" } -solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.18.11", default-features = false } -solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.18.11" } -solana-sdk = { path = "../../sdk", version = "=1.18.11" } -solana-transaction-status = { path = "../../transaction-status", version = "=1.18.11" } -agave-validator = { path = "../../validator", version = "=1.18.11" } -solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.18.11" } +solana-account-decoder = { path = "../../account-decoder", version = "=1.18.12" } +solana-accounts-db = { path = "../../accounts-db", version = "=1.18.12" } +solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.18.12" } +solana-cli-output = { path = "../../cli-output", version = "=1.18.12" } +solana-ledger = { path = "../../ledger", version = "=1.18.12" } +solana-logger = { path = "../../logger", version = "=1.18.12" } +solana-measure = { path = "../../measure", version = "=1.18.12" } +solana-program = { path = "../../sdk/program", version = "=1.18.12" } +solana-program-runtime = { path = "../../program-runtime", version = "=1.18.12" } +solana-program-test = { path = "../../program-test", version = "=1.18.12" } +solana-runtime = { path = "../../runtime", version = "=1.18.12" } +solana-sbf-rust-128bit-dep = { path = "rust/128bit_dep", version = "=1.18.12" } +solana-sbf-rust-invoke = { path = "rust/invoke", version = "=1.18.12" } +solana-sbf-rust-invoked = { path = "rust/invoked", version = "=1.18.12", default-features = false } +solana-sbf-rust-many-args-dep = { path = "rust/many_args_dep", version = "=1.18.12" } +solana-sbf-rust-mem = { path = "rust/mem", version = "=1.18.12" } +solana-sbf-rust-param-passing-dep = { path = "rust/param_passing_dep", version = "=1.18.12" } +solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.18.12", default-features = false } +solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.18.12" } +solana-sdk = { path = "../../sdk", version = "=1.18.12" } +solana-transaction-status = { path = "../../transaction-status", version = "=1.18.12" } +agave-validator = { path = "../../validator", version = "=1.18.12" } +solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.18.12" } solana_rbpf = "=0.8.0" static_assertions = "1.1.0" thiserror = "1.0" diff --git a/quic-client/tests/quic_client.rs b/quic-client/tests/quic_client.rs index 9f18acd5c75772..cbedf9a663ddbc 100644 --- a/quic-client/tests/quic_client.rs +++ b/quic-client/tests/quic_client.rs @@ -10,8 +10,10 @@ mod tests { }, solana_sdk::{net::DEFAULT_TPU_COALESCE, packet::PACKET_DATA_SIZE, signature::Keypair}, solana_streamer::{ - nonblocking::quic::DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, quic::SpawnServerResult, - streamer::StakedNodes, tls_certificates::new_self_signed_tls_certificate, + nonblocking::quic::{DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, + quic::SpawnServerResult, + streamer::StakedNodes, + tls_certificates::new_self_signed_tls_certificate, }, std::{ net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, @@ -83,6 +85,7 @@ mod tests { staked_nodes, 10, 10, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) @@ -152,7 +155,12 @@ mod tests { let (sender, receiver) = unbounded(); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); let (s, exit, keypair, ip) = server_args(); - let (_, _, t) = solana_streamer::nonblocking::quic::spawn_server( + let solana_streamer::nonblocking::quic::SpawnNonBlockingServerResult { + endpoint: _, + stats: _, + thread: t, + max_concurrent_connections: _, + } = solana_streamer::nonblocking::quic::spawn_server( "quic_streamer_test", s.try_clone().unwrap(), &keypair, @@ -163,6 +171,7 @@ mod tests { staked_nodes, 10, 10, + DEFAULT_MAX_STREAMS_PER_MS, Duration::from_secs(1), // wait_for_chunk_timeout DEFAULT_TPU_COALESCE, ) @@ -225,6 +234,7 @@ mod tests { staked_nodes.clone(), 10, 10, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) @@ -253,6 +263,7 @@ mod tests { staked_nodes, 10, 10, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) diff --git a/rpc/src/cluster_tpu_info.rs b/rpc/src/cluster_tpu_info.rs index ad35773216bdcb..c2e515ab5e2b9b 100644 --- a/rpc/src/cluster_tpu_info.rs +++ b/rpc/src/cluster_tpu_info.rs @@ -153,7 +153,6 @@ mod test { bank.clone(), Some((2, 2)), bank.ticks_per_slot(), - &Pubkey::default(), Arc::new(blockstore), &Arc::new(LeaderScheduleCache::new_from_bank(&bank)), &PohConfig::default(), diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 59484817ad06c6..9b892ade4d7766 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -3602,8 +3602,11 @@ pub mod rpc_full { let (wire_transaction, unsanitized_tx) = decode_and_deserialize::(data, binary_encoding)?; - let preflight_commitment = - preflight_commitment.map(|commitment| CommitmentConfig { commitment }); + let preflight_commitment = if skip_preflight { + Some(CommitmentConfig::processed()) + } else { + preflight_commitment.map(|commitment| CommitmentConfig { commitment }) + }; let preflight_bank = &*meta.get_bank_with_config(RpcContextConfig { commitment: preflight_commitment, min_context_slot, @@ -3619,7 +3622,7 @@ pub mod rpc_full { let durable_nonce_info = transaction .get_durable_nonce() .map(|&pubkey| (pubkey, *transaction.message().recent_blockhash())); - if durable_nonce_info.is_some() { + if durable_nonce_info.is_some() || (skip_preflight && last_valid_block_height == 0) { // While it uses a defined constant, this last_valid_block_height value is chosen arbitrarily. // It provides a fallback timeout for durable-nonce transaction retries in case of // malicious packing of the retry queue. Durable-nonce transactions are otherwise diff --git a/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml b/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml index 5350471d950dfc..24699f431cebc5 100644 --- a/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml +++ b/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fail" -version = "1.18.11" +version = "1.18.12" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -10,7 +10,7 @@ edition = "2021" publish = false [dependencies] -solana-program = { path = "../../../../program", version = "=1.18.11" } +solana-program = { path = "../../../../program", version = "=1.18.12" } [lib] crate-type = ["cdylib"] diff --git a/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml b/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml index 3fa04280b36a1c..ad5c0aacffec95 100644 --- a/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml +++ b/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "noop" -version = "1.18.11" +version = "1.18.12" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -10,7 +10,7 @@ edition = "2021" publish = false [dependencies] -solana-program = { path = "../../../../program", version = "=1.18.11" } +solana-program = { path = "../../../../program", version = "=1.18.12" } [lib] crate-type = ["cdylib"] diff --git a/sdk/program/src/alt_bn128/compression.rs b/sdk/program/src/alt_bn128/compression.rs index 2791b8fd35f8f5..0b63b202d4d854 100644 --- a/sdk/program/src/alt_bn128/compression.rs +++ b/sdk/program/src/alt_bn128/compression.rs @@ -20,6 +20,8 @@ mod alt_bn128_compression_size { pub const G2_COMPRESSED: usize = 64; } +// AltBn128CompressionError must be removed once the +// simplify_alt_bn128_syscall_error_codes feature gets activated #[derive(Debug, Error, Clone, PartialEq, Eq)] pub enum AltBn128CompressionError { #[error("Unexpected error")] @@ -51,13 +53,14 @@ impl From for AltBn128CompressionError { impl From for u64 { fn from(v: AltBn128CompressionError) -> u64 { + // note: should never return 0, as it risks to be confused with syscall success match v { AltBn128CompressionError::G1DecompressionFailed => 1, AltBn128CompressionError::G2DecompressionFailed => 2, AltBn128CompressionError::G1CompressionFailed => 3, AltBn128CompressionError::G2CompressionFailed => 4, AltBn128CompressionError::InvalidInputSize => 5, - AltBn128CompressionError::UnexpectedError => 0, + AltBn128CompressionError::UnexpectedError => 6, } } } @@ -118,7 +121,7 @@ mod target_arch { .map_err(|_| AltBn128CompressionError::G1CompressionFailed)?; let mut g1_bytes = [0u8; alt_bn128_compression_size::G1_COMPRESSED]; G1::serialize_compressed(&g1, g1_bytes.as_mut_slice()) - .map_err(|_| AltBn128CompressionError::G2CompressionFailed)?; + .map_err(|_| AltBn128CompressionError::G1CompressionFailed)?; Ok(convert_endianness::<32, 32>(&g1_bytes)) } @@ -131,9 +134,12 @@ mod target_arch { if g2_bytes == [0u8; alt_bn128_compression_size::G2_COMPRESSED] { return Ok([0u8; alt_bn128_compression_size::G2]); } - let decompressed_g2 = - G2::deserialize_compressed(convert_endianness::<64, 64>(&g2_bytes).as_slice()) - .map_err(|_| AltBn128CompressionError::G2DecompressionFailed)?; + let decompressed_g2 = G2::deserialize_with_mode( + convert_endianness::<64, 64>(&g2_bytes).as_slice(), + Compress::Yes, + Validate::No, + ) + .map_err(|_| AltBn128CompressionError::G2DecompressionFailed)?; let mut decompressed_g2_bytes = [0u8; alt_bn128_compression_size::G2]; decompressed_g2 .x @@ -160,7 +166,7 @@ mod target_arch { Compress::No, Validate::No, ) - .map_err(|_| AltBn128CompressionError::G2DecompressionFailed)?; + .map_err(|_| AltBn128CompressionError::G2CompressionFailed)?; let mut g2_bytes = [0u8; alt_bn128_compression_size::G2_COMPRESSED]; G2::serialize_compressed(&g2, g2_bytes.as_mut_slice()) .map_err(|_| AltBn128CompressionError::G2CompressionFailed)?; @@ -205,7 +211,7 @@ mod target_arch { match result { 0 => Ok(result_buffer), - error => Err(AltBn128CompressionError::from(error)), + _ => Err(AltBn128CompressionError::UnexpectedError), } } @@ -222,7 +228,7 @@ mod target_arch { match result { 0 => Ok(result_buffer), - error => Err(AltBn128CompressionError::from(error)), + _ => Err(AltBn128CompressionError::UnexpectedError), } } @@ -241,7 +247,7 @@ mod target_arch { match result { 0 => Ok(result_buffer), - error => Err(AltBn128CompressionError::from(error)), + _ => Err(AltBn128CompressionError::UnexpectedError), } } @@ -260,7 +266,7 @@ mod target_arch { match result { 0 => Ok(result_buffer), - error => Err(AltBn128CompressionError::from(error)), + _ => Err(AltBn128CompressionError::UnexpectedError), } } } diff --git a/sdk/program/src/alt_bn128/mod.rs b/sdk/program/src/alt_bn128/mod.rs index f214157152c114..815632712c3722 100644 --- a/sdk/program/src/alt_bn128/mod.rs +++ b/sdk/program/src/alt_bn128/mod.rs @@ -41,6 +41,8 @@ mod consts { pub const ALT_BN128_PAIRING: u64 = 3; } +// AltBn128Error must be removed once the +// simplify_alt_bn128_syscall_error_codes feature gets activated #[derive(Debug, Error, Clone, PartialEq, Eq)] pub enum AltBn128Error { #[error("The input data is invalid")] @@ -72,13 +74,14 @@ impl From for AltBn128Error { impl From for u64 { fn from(v: AltBn128Error) -> u64 { + // note: should never return 0, as it risks to be confused with syscall success match v { AltBn128Error::InvalidInputData => 1, AltBn128Error::GroupError => 2, AltBn128Error::SliceOutOfBounds => 3, AltBn128Error::TryIntoVecError(_) => 4, AltBn128Error::ProjectiveToG1Failed => 5, - AltBn128Error::UnexpectedError => 0, + AltBn128Error::UnexpectedError => 6, } } } @@ -319,7 +322,7 @@ mod target_arch { match result { 0 => Ok(result_buffer.to_vec()), - error => Err(AltBn128Error::from(error)), + _ => Err(AltBn128Error::UnexpectedError), } } @@ -339,7 +342,7 @@ mod target_arch { match result { 0 => Ok(result_buffer.to_vec()), - error => Err(AltBn128Error::from(error)), + _ => Err(AltBn128Error::UnexpectedError), } } @@ -363,7 +366,7 @@ mod target_arch { match result { 0 => Ok(result_buffer.to_vec()), - error => Err(AltBn128Error::from(error)), + _ => Err(AltBn128Error::UnexpectedError), } } } diff --git a/sdk/program/src/poseidon.rs b/sdk/program/src/poseidon.rs index 9c02fe90bc8b50..9e782fa5e85fe7 100644 --- a/sdk/program/src/poseidon.rs +++ b/sdk/program/src/poseidon.rs @@ -7,6 +7,8 @@ use thiserror::Error; /// Length of Poseidon hash result. pub const HASH_BYTES: usize = 32; +// PoseidonSyscallError must be removed once the +// simplify_alt_bn128_syscall_error_codes feature gets activated #[derive(Error, Debug)] pub enum PoseidonSyscallError { #[error("Invalid parameters.")] @@ -267,7 +269,7 @@ pub fn hashv( match result { 0 => Ok(PoseidonHash::new(hash_result)), - e => Err(PoseidonSyscallError::from(e)), + _ => Err(PoseidonSyscallError::Unexpected), } } } diff --git a/sdk/src/feature_set.rs b/sdk/src/feature_set.rs index 716b268526191f..32c9973c2eb946 100644 --- a/sdk/src/feature_set.rs +++ b/sdk/src/feature_set.rs @@ -561,6 +561,11 @@ pub mod enable_bpf_loader_set_authority_checked_ix { pub mod enable_alt_bn128_syscall { solana_sdk::declare_id!("A16q37opZdQMCbe5qJ6xpBB9usykfv8jZaMkxvZQi4GJ"); } + +pub mod simplify_alt_bn128_syscall_error_codes { + solana_sdk::declare_id!("JDn5q3GBeqzvUa7z67BbmVHVdE3EbUAjvFep3weR3jxX"); +} + pub mod enable_alt_bn128_compression_syscall { solana_sdk::declare_id!("EJJewYSddEEtSZHiqugnvhQHiWyZKjkFDQASd7oKSagn"); } @@ -772,6 +777,10 @@ pub mod enable_gossip_duplicate_proof_ingestion { solana_sdk::declare_id!("FNKCMBzYUdjhHyPdsKG2LSmdzH8TCHXn3ytj8RNBS4nG"); } +pub mod chained_merkle_conflict_duplicate_proofs { + solana_sdk::declare_id!("chaie9S2zVfuxJKNRGkyTDokLwWxx6kD2ZLsqQHaDD8"); +} + pub mod enable_chained_merkle_shreds { solana_sdk::declare_id!("7uZBkJXJ1HkuP6R3MJfZs7mLwymBcDbKdqbF51ZWLier"); } @@ -914,6 +923,7 @@ lazy_static! { (check_syscall_outputs_do_not_overlap::id(), "check syscall outputs do_not overlap #28600"), (enable_bpf_loader_set_authority_checked_ix::id(), "enable bpf upgradeable loader SetAuthorityChecked instruction #28424"), (enable_alt_bn128_syscall::id(), "add alt_bn128 syscalls #27961"), + (simplify_alt_bn128_syscall_error_codes::id(), "simplify alt_bn128 syscall error codes SIMD-0129"), (enable_program_redeployment_cooldown::id(), "enable program redeployment cooldown #29135"), (commission_updates_only_allowed_in_first_half_of_epoch::id(), "validator commission updates are only allowed in the first half of an epoch #29362"), (enable_turbine_fanout_experiments::id(), "enable turbine fanout experiments #29393"), @@ -970,6 +980,7 @@ lazy_static! { (enable_gossip_duplicate_proof_ingestion::id(), "enable gossip duplicate proof ingestion #32963"), (enable_chained_merkle_shreds::id(), "Enable chained Merkle shreds #34916"), (deprecate_unused_legacy_vote_plumbing::id(), "Deprecate unused legacy vote tx plumbing"), + (chained_merkle_conflict_duplicate_proofs::id(), "generate duplicate proofs for chained merkle root conflicts"), /*************** ADD NEW FEATURES HERE ***************/ ] .iter() diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 14437cdf8acb8a..d0622e278e11c1 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -1,8 +1,8 @@ use { crate::{ nonblocking::stream_throttle::{ - ConnectionStreamCounter, StakedStreamLoadEMA, MAX_STREAMS_PER_MS, - STREAM_STOP_CODE_THROTTLING, STREAM_THROTTLING_INTERVAL_MS, + ConnectionStreamCounter, StakedStreamLoadEMA, STREAM_THROTTLING_INTERVAL, + STREAM_THROTTLING_INTERVAL_MS, }, quic::{configure_server, QuicServerError, StreamStats}, streamer::StakedNodes, @@ -53,7 +53,7 @@ use { // introduce any other awaits while holding the RwLock. sync::{Mutex, MutexGuard}, task::JoinHandle, - time::timeout, + time::{sleep, timeout}, }, }; @@ -74,6 +74,9 @@ const CONNECTION_CLOSE_REASON_EXCEED_MAX_STREAM_COUNT: &[u8] = b"exceed_max_stre const CONNECTION_CLOSE_CODE_TOO_MANY: u32 = 4; const CONNECTION_CLOSE_REASON_TOO_MANY: &[u8] = b"too_many"; +/// Limit to 250K PPS +pub const DEFAULT_MAX_STREAMS_PER_MS: u64 = 250; + // A sequence of bytes that is part of a packet // along with where in the packet it is struct PacketChunk { @@ -110,6 +113,13 @@ impl ConnectionPeerType { } } +pub struct SpawnNonBlockingServerResult { + pub endpoint: Endpoint, + pub stats: Arc, + pub thread: JoinHandle<()>, + pub max_concurrent_connections: usize, +} + #[allow(clippy::too_many_arguments)] pub fn spawn_server( name: &'static str, @@ -122,11 +132,14 @@ pub fn spawn_server( staked_nodes: Arc>, max_staked_connections: usize, max_unstaked_connections: usize, + max_streams_per_ms: u64, wait_for_chunk_timeout: Duration, coalesce: Duration, -) -> Result<(Endpoint, Arc, JoinHandle<()>), QuicServerError> { +) -> Result { info!("Start {name} quic server on {sock:?}"); - let (config, _cert) = configure_server(keypair, gossip_host)?; + let concurrent_connections = max_staked_connections + max_unstaked_connections; + let max_concurrent_connections = concurrent_connections + concurrent_connections / 4; + let (config, _cert) = configure_server(keypair, gossip_host, max_concurrent_connections)?; let endpoint = Endpoint::new( EndpointConfig::default(), @@ -135,6 +148,7 @@ pub fn spawn_server( Arc::new(TokioRuntime), ) .map_err(QuicServerError::EndpointFailed)?; + let stats = Arc::::default(); let handle = tokio::spawn(run_server( name, @@ -145,11 +159,17 @@ pub fn spawn_server( staked_nodes, max_staked_connections, max_unstaked_connections, + max_streams_per_ms, stats.clone(), wait_for_chunk_timeout, coalesce, )); - Ok((endpoint, stats, handle)) + Ok(SpawnNonBlockingServerResult { + endpoint, + stats, + thread: handle, + max_concurrent_connections, + }) } #[allow(clippy::too_many_arguments)] @@ -162,6 +182,7 @@ async fn run_server( staked_nodes: Arc>, max_staked_connections: usize, max_unstaked_connections: usize, + max_streams_per_ms: u64, stats: Arc, wait_for_chunk_timeout: Duration, coalesce: Duration, @@ -172,8 +193,9 @@ async fn run_server( let unstaked_connection_table: Arc> = Arc::new(Mutex::new(ConnectionTable::new())); let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( - max_unstaked_connections > 0, stats.clone(), + max_unstaked_connections, + max_streams_per_ms, )); let staked_connection_table: Arc> = Arc::new(Mutex::new(ConnectionTable::new())); @@ -204,6 +226,7 @@ async fn run_server( staked_nodes.clone(), max_staked_connections, max_unstaked_connections, + max_streams_per_ms, stats.clone(), wait_for_chunk_timeout, stream_load_ema.clone(), @@ -337,16 +360,10 @@ fn handle_and_cache_new_connection( params.total_stake, ) as u64) { - connection.set_max_concurrent_uni_streams(max_uni_streams); + let remote_addr = connection.remote_address(); let receive_window = compute_recieve_window(params.max_stake, params.min_stake, params.peer_type); - if let Ok(receive_window) = receive_window { - connection.set_receive_window(receive_window); - } - - let remote_addr = connection.remote_address(); - debug!( "Peer type {:?}, total stake {}, max streams {} receive_window {:?} from peer {}", params.peer_type, @@ -367,6 +384,12 @@ fn handle_and_cache_new_connection( ) { drop(connection_table_l); + + if let Ok(receive_window) = receive_window { + connection.set_receive_window(receive_window); + } + connection.set_max_concurrent_uni_streams(max_uni_streams); + tokio::spawn(handle_connection( connection, remote_addr, @@ -482,6 +505,7 @@ async fn setup_connection( staked_nodes: Arc>, max_staked_connections: usize, max_unstaked_connections: usize, + max_streams_per_ms: u64, stats: Arc, wait_for_chunk_timeout: Duration, stream_load_ema: Arc, @@ -503,7 +527,7 @@ async fn setup_connection( // The heuristic is that the stake should be large engouh to have 1 stream pass throuh within one throttle // interval during which we allow max (MAX_STREAMS_PER_MS * STREAM_THROTTLING_INTERVAL_MS) streams. let min_stake_ratio = - 1_f64 / (MAX_STREAMS_PER_MS * STREAM_THROTTLING_INTERVAL_MS) as f64; + 1_f64 / (max_streams_per_ms * STREAM_THROTTLING_INTERVAL_MS) as f64; let stake_ratio = stake as f64 / total_stake as f64; let peer_type = if stake_ratio < min_stake_ratio { // If it is a staked connection with ultra low stake ratio, treat it as unstaked. @@ -759,25 +783,36 @@ async fn handle_connection( params.total_stake, ); - stream_counter.reset_throttling_params_if_needed(); - if stream_counter.stream_count.load(Ordering::Relaxed) - >= max_streams_per_throttling_interval - { - stats.throttled_streams.fetch_add(1, Ordering::Relaxed); - match params.peer_type { - ConnectionPeerType::Unstaked => { - stats - .throttled_unstaked_streams - .fetch_add(1, Ordering::Relaxed); - } - ConnectionPeerType::Staked(_) => { - stats - .throttled_staked_streams - .fetch_add(1, Ordering::Relaxed); + let throttle_interval_start = + stream_counter.reset_throttling_params_if_needed(); + let streams_read_in_throttle_interval = + stream_counter.stream_count.load(Ordering::Relaxed); + if streams_read_in_throttle_interval >= max_streams_per_throttling_interval { + // The peer is sending faster than we're willing to read. Sleep for what's + // left of this read interval so the peer backs off. + let throttle_duration = STREAM_THROTTLING_INTERVAL + .saturating_sub(throttle_interval_start.elapsed()); + + if !throttle_duration.is_zero() { + debug!("Throttling stream from {remote_addr:?}, peer type: {:?}, total stake: {}, \ + max_streams_per_interval: {max_streams_per_throttling_interval}, read_interval_streams: {streams_read_in_throttle_interval} \ + throttle_duration: {throttle_duration:?}", + params.peer_type, params.total_stake); + stats.throttled_streams.fetch_add(1, Ordering::Relaxed); + match params.peer_type { + ConnectionPeerType::Unstaked => { + stats + .throttled_unstaked_streams + .fetch_add(1, Ordering::Relaxed); + } + ConnectionPeerType::Staked(_) => { + stats + .throttled_staked_streams + .fetch_add(1, Ordering::Relaxed); + } } + sleep(throttle_duration).await; } - let _ = stream.stop(VarInt::from_u32(STREAM_STOP_CODE_THROTTLING)); - continue; } stream_load_ema.increment_load(params.peer_type); stream_counter.stream_count.fetch_add(1, Ordering::Relaxed); @@ -1127,16 +1162,10 @@ impl ConnectionTable { if has_connection_capacity { let exit = Arc::new(AtomicBool::new(false)); let last_update = Arc::new(AtomicU64::new(last_update)); - let stream_counter = if peer_type.is_staked() { - connection_entry - .first() - .map(|entry| entry.stream_counter.clone()) - .unwrap_or(Arc::new(ConnectionStreamCounter::new())) - } else { - // Unstaked connections are tracked using peer IP address. It's possible that different clients - // use the same IP due to NAT. So counting all the streams from a given IP could be too restrictive. - Arc::new(ConnectionStreamCounter::new()) - }; + let stream_counter = connection_entry + .first() + .map(|entry| entry.stream_counter.clone()) + .unwrap_or(Arc::new(ConnectionStreamCounter::new())); connection_entry.push(ConnectionEntry::new( exit.clone(), peer_type, @@ -1277,7 +1306,12 @@ pub mod test { let ip = "127.0.0.1".parse().unwrap(); let server_address = s.local_addr().unwrap(); let staked_nodes = Arc::new(RwLock::new(option_staked_nodes.unwrap_or_default())); - let (_, stats, t) = spawn_server( + let SpawnNonBlockingServerResult { + endpoint: _, + stats, + thread: t, + max_concurrent_connections: _, + } = spawn_server( "quic_streamer_test", s, &keypair, @@ -1288,6 +1322,7 @@ pub mod test { staked_nodes, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, Duration::from_secs(2), DEFAULT_TPU_COALESCE, ) @@ -1713,7 +1748,12 @@ pub mod test { let ip = "127.0.0.1".parse().unwrap(); let server_address = s.local_addr().unwrap(); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); - let (_, _, t) = spawn_server( + let SpawnNonBlockingServerResult { + endpoint: _, + stats: _, + thread: t, + max_concurrent_connections: _, + } = spawn_server( "quic_streamer_test", s, &keypair, @@ -1724,6 +1764,7 @@ pub mod test { staked_nodes, MAX_STAKED_CONNECTIONS, 0, // Do not allow any connection from unstaked clients/nodes + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) @@ -1744,7 +1785,12 @@ pub mod test { let ip = "127.0.0.1".parse().unwrap(); let server_address = s.local_addr().unwrap(); let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); - let (_, stats, t) = spawn_server( + let SpawnNonBlockingServerResult { + endpoint: _, + stats, + thread: t, + max_concurrent_connections: _, + } = spawn_server( "quic_streamer_test", s, &keypair, @@ -1755,6 +1801,7 @@ pub mod test { staked_nodes, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) diff --git a/streamer/src/nonblocking/stream_throttle.rs b/streamer/src/nonblocking/stream_throttle.rs index 95ba1cb550538e..699d8d7faf33fb 100644 --- a/streamer/src/nonblocking/stream_throttle.rs +++ b/streamer/src/nonblocking/stream_throttle.rs @@ -1,8 +1,5 @@ use { - crate::{ - nonblocking::quic::ConnectionPeerType, - quic::{StreamStats, MAX_UNSTAKED_CONNECTIONS}, - }, + crate::{nonblocking::quic::ConnectionPeerType, quic::StreamStats}, percentage::Percentage, std::{ cmp, @@ -14,11 +11,10 @@ use { }, }; -/// Limit to 250K PPS -pub const MAX_STREAMS_PER_MS: u64 = 250; const MAX_UNSTAKED_STREAMS_PERCENT: u64 = 20; pub const STREAM_THROTTLING_INTERVAL_MS: u64 = 100; -pub const STREAM_STOP_CODE_THROTTLING: u32 = 15; +pub const STREAM_THROTTLING_INTERVAL: Duration = + Duration::from_millis(STREAM_THROTTLING_INTERVAL_MS); const STREAM_LOAD_EMA_INTERVAL_MS: u64 = 5; const STREAM_LOAD_EMA_INTERVAL_COUNT: u64 = 10; const EMA_WINDOW_MS: u64 = STREAM_LOAD_EMA_INTERVAL_MS * STREAM_LOAD_EMA_INTERVAL_COUNT; @@ -38,27 +34,36 @@ pub(crate) struct StakedStreamLoadEMA { } impl StakedStreamLoadEMA { - pub(crate) fn new(allow_unstaked_streams: bool, stats: Arc) -> Self { + pub(crate) fn new( + stats: Arc, + max_unstaked_connections: usize, + max_streams_per_ms: u64, + ) -> Self { + let allow_unstaked_streams = max_unstaked_connections > 0; let max_staked_load_in_ema_window = if allow_unstaked_streams { - (MAX_STREAMS_PER_MS - - Percentage::from(MAX_UNSTAKED_STREAMS_PERCENT).apply_to(MAX_STREAMS_PER_MS)) + (max_streams_per_ms + - Percentage::from(MAX_UNSTAKED_STREAMS_PERCENT).apply_to(max_streams_per_ms)) * EMA_WINDOW_MS } else { - MAX_STREAMS_PER_MS * EMA_WINDOW_MS + max_streams_per_ms * EMA_WINDOW_MS }; let max_num_unstaked_connections = - u64::try_from(MAX_UNSTAKED_CONNECTIONS).unwrap_or_else(|_| { + u64::try_from(max_unstaked_connections).unwrap_or_else(|_| { error!( "Failed to convert maximum number of unstaked connections {} to u64.", - MAX_UNSTAKED_CONNECTIONS + max_unstaked_connections ); 500 }); - let max_unstaked_load_in_throttling_window = Percentage::from(MAX_UNSTAKED_STREAMS_PERCENT) - .apply_to(MAX_STREAMS_PER_MS * STREAM_THROTTLING_INTERVAL_MS) - .saturating_div(max_num_unstaked_connections); + let max_unstaked_load_in_throttling_window = if allow_unstaked_streams { + Percentage::from(MAX_UNSTAKED_STREAMS_PERCENT) + .apply_to(max_streams_per_ms * STREAM_THROTTLING_INTERVAL_MS) + .saturating_div(max_num_unstaked_connections) + } else { + 0 + }; Self { current_load_ema: AtomicU64::default(), @@ -204,19 +209,24 @@ impl ConnectionStreamCounter { } } - pub(crate) fn reset_throttling_params_if_needed(&self) { - const THROTTLING_INTERVAL: Duration = Duration::from_millis(STREAM_THROTTLING_INTERVAL_MS); - if tokio::time::Instant::now().duration_since(*self.last_throttling_instant.read().unwrap()) - > THROTTLING_INTERVAL + /// Reset the counter and last throttling instant and + /// return last_throttling_instant regardless it is reset or not. + pub(crate) fn reset_throttling_params_if_needed(&self) -> tokio::time::Instant { + let last_throttling_instant = *self.last_throttling_instant.read().unwrap(); + if tokio::time::Instant::now().duration_since(last_throttling_instant) + > STREAM_THROTTLING_INTERVAL { let mut last_throttling_instant = self.last_throttling_instant.write().unwrap(); // Recheck as some other thread might have done throttling since this thread tried to acquire the write lock. if tokio::time::Instant::now().duration_since(*last_throttling_instant) - > THROTTLING_INTERVAL + > STREAM_THROTTLING_INTERVAL { *last_throttling_instant = tokio::time::Instant::now(); self.stream_count.store(0, Ordering::Relaxed); } + *last_throttling_instant + } else { + last_throttling_instant } } } @@ -225,7 +235,12 @@ impl ConnectionStreamCounter { pub mod test { use { super::*, - crate::{nonblocking::stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS, quic::StreamStats}, + crate::{ + nonblocking::{ + quic::DEFAULT_MAX_STREAMS_PER_MS, stream_throttle::STREAM_LOAD_EMA_INTERVAL_MS, + }, + quic::{StreamStats, MAX_UNSTAKED_CONNECTIONS}, + }, std::{ sync::{atomic::Ordering, Arc}, time::{Duration, Instant}, @@ -235,8 +250,9 @@ pub mod test { #[test] fn test_max_streams_for_unstaked_connection() { let load_ema = Arc::new(StakedStreamLoadEMA::new( - true, Arc::new(StreamStats::default()), + MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, )); // 25K packets per ms * 20% / 500 max unstaked connections assert_eq!( @@ -251,8 +267,9 @@ pub mod test { #[test] fn test_max_streams_for_staked_connection() { let load_ema = Arc::new(StakedStreamLoadEMA::new( - true, Arc::new(StreamStats::default()), + MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, )); // EMA load is used for staked connections to calculate max number of allowed streams. @@ -342,8 +359,9 @@ pub mod test { #[test] fn test_max_streams_for_staked_connection_with_no_unstaked_connections() { let load_ema = Arc::new(StakedStreamLoadEMA::new( - false, Arc::new(StreamStats::default()), + 0, + DEFAULT_MAX_STREAMS_PER_MS, )); // EMA load is used for staked connections to calculate max number of allowed streams. @@ -413,12 +431,12 @@ pub mod test { 10000 ); - // At 1/40000 stake weight, and minimum load, it should still allow + // At 1/400000 stake weight, and minimum load, it should still allow // max_unstaked_load_in_throttling_window + 1 streams. assert_eq!( load_ema.available_load_capacity_in_throttling_duration( ConnectionPeerType::Staked(1), - 40000 + 400000 ), load_ema .max_unstaked_load_in_throttling_window @@ -429,8 +447,9 @@ pub mod test { #[test] fn test_update_ema() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( - true, Arc::new(StreamStats::default()), + MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema .load_in_recent_interval @@ -457,8 +476,9 @@ pub mod test { #[test] fn test_update_ema_missing_interval() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( - true, Arc::new(StreamStats::default()), + MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema .load_in_recent_interval @@ -476,8 +496,9 @@ pub mod test { #[test] fn test_update_ema_if_needed() { let stream_load_ema = Arc::new(StakedStreamLoadEMA::new( - true, Arc::new(StreamStats::default()), + MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, )); stream_load_ema .load_in_recent_interval diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 78ccbeb6e60a13..c076f76d8704c3 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -62,6 +62,7 @@ impl rustls::server::ClientCertVerifier for SkipClientVerification { pub(crate) fn configure_server( identity_keypair: &Keypair, gossip_host: IpAddr, + max_concurrent_connections: usize, ) -> Result<(ServerConfig, String), QuicServerError> { let (cert, priv_key) = new_self_signed_tls_certificate(identity_keypair, gossip_host)?; let cert_chain_pem_parts = vec![Pem { @@ -77,6 +78,7 @@ pub(crate) fn configure_server( server_tls_config.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; let mut server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); + server_config.concurrent_connections(max_concurrent_connections as u32); server_config.use_retry(true); let config = Arc::get_mut(&mut server_config.transport).unwrap(); @@ -85,11 +87,7 @@ pub(crate) fn configure_server( (QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS.saturating_mul(2)) as u32; config.max_concurrent_uni_streams(MAX_CONCURRENT_UNI_STREAMS.into()); config.stream_receive_window((PACKET_DATA_SIZE as u32).into()); - config.receive_window( - (PACKET_DATA_SIZE as u32) - .saturating_mul(MAX_CONCURRENT_UNI_STREAMS) - .into(), - ); + config.receive_window((PACKET_DATA_SIZE as u32).into()); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); config.max_idle_timeout(Some(timeout)); @@ -122,11 +120,12 @@ pub enum QuicServerError { pub struct EndpointKeyUpdater { endpoint: Endpoint, gossip_host: IpAddr, + max_concurrent_connections: usize, } impl NotifyKeyUpdate for EndpointKeyUpdater { fn update_key(&self, key: &Keypair) -> Result<(), Box> { - let (config, _) = configure_server(key, self.gossip_host)?; + let (config, _) = configure_server(key, self.gossip_host, self.max_concurrent_connections)?; self.endpoint.set_server_config(Some(config)); Ok(()) } @@ -471,11 +470,12 @@ pub fn spawn_server( staked_nodes: Arc>, max_staked_connections: usize, max_unstaked_connections: usize, + max_streams_per_ms: u64, wait_for_chunk_timeout: Duration, coalesce: Duration, ) -> Result { let runtime = rt(); - let (endpoint, _stats, task) = { + let result = { let _guard = runtime.enter(); crate::nonblocking::quic::spawn_server( name, @@ -488,6 +488,7 @@ pub fn spawn_server( staked_nodes, max_staked_connections, max_unstaked_connections, + max_streams_per_ms, wait_for_chunk_timeout, coalesce, ) @@ -495,17 +496,20 @@ pub fn spawn_server( let handle = thread::Builder::new() .name("solQuicServer".into()) .spawn(move || { - if let Err(e) = runtime.block_on(task) { + if let Err(e) = runtime.block_on(result.thread) { warn!("error from runtime.block_on: {:?}", e); } }) .unwrap(); + let updater = EndpointKeyUpdater { - endpoint: endpoint.clone(), + endpoint: result.endpoint.clone(), gossip_host, + max_concurrent_connections: result.max_concurrent_connections, }; + Ok(SpawnServerResult { - endpoint, + endpoint: result.endpoint, thread: handle, key_updater: Arc::new(updater), }) @@ -515,7 +519,9 @@ pub fn spawn_server( mod test { use { super::*, - crate::nonblocking::quic::{test::*, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, + crate::nonblocking::quic::{ + test::*, DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, + }, crossbeam_channel::unbounded, solana_sdk::net::DEFAULT_TPU_COALESCE, std::net::SocketAddr, @@ -549,6 +555,7 @@ mod test { staked_nodes, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) @@ -609,6 +616,7 @@ mod test { staked_nodes, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS, + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) @@ -656,6 +664,7 @@ mod test { staked_nodes, MAX_STAKED_CONNECTIONS, 0, // Do not allow any connection from unstaked clients/nodes + DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT, DEFAULT_TPU_COALESCE, ) diff --git a/tpu-client/src/tpu_client.rs b/tpu-client/src/tpu_client.rs index 9d5a159686d426..6b1e7dbe44f7ac 100644 --- a/tpu-client/src/tpu_client.rs +++ b/tpu-client/src/tpu_client.rs @@ -21,7 +21,11 @@ use { pub const DEFAULT_TPU_ENABLE_UDP: bool = false; pub const DEFAULT_TPU_USE_QUIC: bool = true; -pub const DEFAULT_TPU_CONNECTION_POOL_SIZE: usize = 4; + +/// The default connection count is set to 1 -- it should +/// be sufficient for most use cases. Validators can use +/// --tpu-connection-pool-size to override this default value. +pub const DEFAULT_TPU_CONNECTION_POOL_SIZE: usize = 1; pub type Result = std::result::Result; diff --git a/transaction-dos/src/main.rs b/transaction-dos/src/main.rs index 8fab7612560585..1e45a1fa5ccda4 100644 --- a/transaction-dos/src/main.rs +++ b/transaction-dos/src/main.rs @@ -249,6 +249,7 @@ fn run_transactions_dos( max_len: None, compute_unit_price: None, max_sign_attempts: 5, + use_rpc: false, skip_fee_check: true, // skip_fee_check }); diff --git a/turbine/benches/cluster_nodes.rs b/turbine/benches/cluster_nodes.rs index 66215380140e4d..4e46ff28b7df04 100644 --- a/turbine/benches/cluster_nodes.rs +++ b/turbine/benches/cluster_nodes.rs @@ -6,7 +6,7 @@ use { rand::{seq::SliceRandom, Rng}, solana_gossip::legacy_contact_info::LegacyContactInfo as ContactInfo, solana_ledger::shred::{Shred, ShredFlags}, - solana_sdk::{clock::Slot, pubkey::Pubkey}, + solana_sdk::{clock::Slot, genesis_config::ClusterType, pubkey::Pubkey}, solana_turbine::{ cluster_nodes::{make_test_cluster, new_cluster_nodes, ClusterNodes}, retransmit_stage::RetransmitStage, @@ -21,7 +21,8 @@ fn make_cluster_nodes( unstaked_ratio: Option<(u32, u32)>, ) -> (Vec, ClusterNodes) { let (nodes, stakes, cluster_info) = make_test_cluster(rng, 5_000, unstaked_ratio); - let cluster_nodes = new_cluster_nodes::(&cluster_info, &stakes); + let cluster_nodes = + new_cluster_nodes::(&cluster_info, ClusterType::Development, &stakes); (nodes, cluster_nodes) } diff --git a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs index adca69ed4938cd..2fd7dbf3b9e0fd 100644 --- a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs +++ b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs @@ -237,9 +237,11 @@ impl BroadcastRun for BroadcastDuplicatesRun { sigs, ); - assert_eq!(original_last_data_shred.len(), 1); - assert_eq!(partition_last_data_shred.len(), 1); - self.next_shred_index += 1; + assert_eq!( + original_last_data_shred.len(), + partition_last_data_shred.len() + ); + self.next_shred_index += u32::try_from(original_last_data_shred.len()).unwrap(); (original_last_data_shred, partition_last_data_shred) }); diff --git a/turbine/src/cluster_nodes.rs b/turbine/src/cluster_nodes.rs index 9dd122d85c3df7..76bae78bc1e6c9 100644 --- a/turbine/src/cluster_nodes.rs +++ b/turbine/src/cluster_nodes.rs @@ -17,6 +17,7 @@ use { solana_sdk::{ clock::{Epoch, Slot}, feature_set, + genesis_config::ClusterType, native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, signature::{Keypair, Signer}, @@ -29,7 +30,7 @@ use { collections::HashMap, iter::repeat_with, marker::PhantomData, - net::SocketAddr, + net::{IpAddr, SocketAddr}, sync::{Arc, Mutex, RwLock}, time::{Duration, Instant}, }, @@ -39,6 +40,9 @@ use { const DATA_PLANE_FANOUT: usize = 200; pub(crate) const MAX_NUM_TURBINE_HOPS: usize = 4; +// Limit number of nodes per IP address. +const MAX_NUM_NODES_PER_IP_ADDRESS: usize = 10; + #[derive(Debug, Error)] pub enum Error { #[error("Loopback from slot leader: {leader}, shred: {shred:?}")] @@ -81,9 +85,6 @@ pub struct ClusterNodesCache { pub struct RetransmitPeers<'a> { root_distance: usize, // distance from the root node children: Vec<&'a Node>, - // Maps tvu addresses to the first node - // in the shuffle with the same address. - addrs: HashMap, // tvu addresses } impl Node { @@ -147,8 +148,12 @@ impl ClusterNodes { } impl ClusterNodes { - pub fn new(cluster_info: &ClusterInfo, stakes: &HashMap) -> Self { - new_cluster_nodes(cluster_info, stakes) + pub fn new( + cluster_info: &ClusterInfo, + cluster_type: ClusterType, + stakes: &HashMap, + ) -> Self { + new_cluster_nodes(cluster_info, cluster_type, stakes) } pub(crate) fn get_broadcast_peer(&self, shred: &ShredId) -> Option<&ContactInfo> { @@ -168,16 +173,13 @@ impl ClusterNodes { let RetransmitPeers { root_distance, children, - addrs, } = self.get_retransmit_peers(slot_leader, shred, fanout)?; let protocol = get_broadcast_protocol(shred); - let peers = children.into_iter().filter_map(|node| { - node.contact_info()? - .tvu(protocol) - .ok() - .filter(|addr| addrs.get(addr) == Some(&node.pubkey())) - }); - Ok((root_distance, peers.collect())) + let peers = children + .into_iter() + .filter_map(|node| node.contact_info()?.tvu(protocol).ok()) + .collect(); + Ok((root_distance, peers)) } pub fn get_retransmit_peers( @@ -197,19 +199,10 @@ impl ClusterNodes { if let Some(index) = self.index.get(slot_leader) { weighted_shuffle.remove_index(*index); } - let mut addrs = HashMap::::with_capacity(self.nodes.len()); let mut rng = get_seeded_rng(slot_leader, shred); - let protocol = get_broadcast_protocol(shred); let nodes: Vec<_> = weighted_shuffle .shuffle(&mut rng) .map(|index| &self.nodes[index]) - .inspect(|node| { - if let Some(node) = node.contact_info() { - if let Ok(addr) = node.tvu(protocol) { - addrs.entry(addr).or_insert(*node.pubkey()); - } - } - }) .collect(); let self_index = nodes .iter() @@ -228,7 +221,6 @@ impl ClusterNodes { Ok(RetransmitPeers { root_distance, children: peers.collect(), - addrs, }) } @@ -272,10 +264,11 @@ impl ClusterNodes { pub fn new_cluster_nodes( cluster_info: &ClusterInfo, + cluster_type: ClusterType, stakes: &HashMap, ) -> ClusterNodes { let self_pubkey = cluster_info.id(); - let nodes = get_nodes(cluster_info, stakes); + let nodes = get_nodes(cluster_info, cluster_type, stakes); let index: HashMap<_, _> = nodes .iter() .enumerate() @@ -298,8 +291,21 @@ pub fn new_cluster_nodes( // All staked nodes + other known tvu-peers + the node itself; // sorted by (stake, pubkey) in descending order. -fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap) -> Vec { +fn get_nodes( + cluster_info: &ClusterInfo, + cluster_type: ClusterType, + stakes: &HashMap, +) -> Vec { let self_pubkey = cluster_info.id(); + let should_dedup_addrs = match cluster_type { + ClusterType::Development => false, + ClusterType::Devnet | ClusterType::Testnet | ClusterType::MainnetBeta => true, + }; + // Maps IP addresses to number of nodes at that IP address. + let mut counts = { + let capacity = if should_dedup_addrs { stakes.len() } else { 0 }; + HashMap::::with_capacity(capacity) + }; // The local node itself. std::iter::once({ let stake = stakes.get(&self_pubkey).copied().unwrap_or_default(); @@ -328,6 +334,30 @@ fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap) -> Vec 0u64).then(|| Node { + node: NodeId::from(node.pubkey()), + stake: node.stake, + }) + } + }) .collect() } @@ -445,6 +475,7 @@ impl ClusterNodesCache { } let nodes = Arc::new(new_cluster_nodes::( cluster_info, + root_bank.cluster_type(), &epoch_staked_nodes.unwrap_or_default(), )); *entry = Some((Instant::now(), Arc::clone(&nodes))); @@ -594,7 +625,8 @@ mod tests { let (nodes, stakes, cluster_info) = make_test_cluster(&mut rng, 1_000, None); // ClusterInfo::tvu_peers excludes the node itself. assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1); - let cluster_nodes = new_cluster_nodes::(&cluster_info, &stakes); + let cluster_nodes = + new_cluster_nodes::(&cluster_info, ClusterType::Development, &stakes); // All nodes with contact-info should be in the index. // Staked nodes with no contact-info should be included. assert!(cluster_nodes.nodes.len() > nodes.len()); @@ -629,7 +661,8 @@ mod tests { let (nodes, stakes, cluster_info) = make_test_cluster(&mut rng, 1_000, None); // ClusterInfo::tvu_peers excludes the node itself. assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1); - let cluster_nodes = ClusterNodes::::new(&cluster_info, &stakes); + let cluster_nodes = + ClusterNodes::::new(&cluster_info, ClusterType::Development, &stakes); // All nodes with contact-info should be in the index. // Excluding this node itself. // Staked nodes with no contact-info should be included.