From a443ff45c5877c2881f838fb61899e6dde0a8e19 Mon Sep 17 00:00:00 2001 From: Bruce Riley Date: Wed, 19 Mar 2025 13:14:32 -0500 Subject: [PATCH] Node: Add reobservation metric --- node/pkg/watchers/algorand/watcher.go | 4 ++++ node/pkg/watchers/aptos/watcher.go | 3 +++ node/pkg/watchers/cosmwasm/watcher.go | 2 ++ node/pkg/watchers/evm/watcher.go | 9 +++++---- node/pkg/watchers/ibc/watcher.go | 4 ++++ node/pkg/watchers/near/tx_processing.go | 5 +++++ node/pkg/watchers/solana/client.go | 4 ++++ node/pkg/watchers/solana/shim.go | 4 ++++ node/pkg/watchers/sui/watcher.go | 4 ++++ node/pkg/watchers/watchers.go | 10 ++++++++++ 10 files changed, 45 insertions(+), 4 deletions(-) diff --git a/node/pkg/watchers/algorand/watcher.go b/node/pkg/watchers/algorand/watcher.go index e295e2e3ce..015f7c5ead 100644 --- a/node/pkg/watchers/algorand/watcher.go +++ b/node/pkg/watchers/algorand/watcher.go @@ -17,6 +17,7 @@ import ( gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1" "github.com/certusone/wormhole/node/pkg/readiness" "github.com/certusone/wormhole/node/pkg/supervisor" + "github.com/certusone/wormhole/node/pkg/watchers" eth_common "github.com/ethereum/go-ethereum/common" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -169,6 +170,9 @@ func lookAtTxn(e *Watcher, t types.SignedTxnInBlock, b types.Block, logger *zap. } algorandMessagesConfirmed.Inc() + if isReobservation { + watchers.ReobservationsByChain.WithLabelValues("algorand", "std").Inc() + } logger.Info("message observed", zap.Time("timestamp", observation.Timestamp), diff --git a/node/pkg/watchers/aptos/watcher.go b/node/pkg/watchers/aptos/watcher.go index f3e5228e85..0d1f29eb32 100644 --- a/node/pkg/watchers/aptos/watcher.go +++ b/node/pkg/watchers/aptos/watcher.go @@ -337,6 +337,9 @@ func (e *Watcher) observeData(logger *zap.Logger, data gjson.Result, nativeSeq u } aptosMessagesConfirmed.WithLabelValues(e.networkID).Inc() + if isReobservation { + watchers.ReobservationsByChain.WithLabelValues(e.chainID.String(), "std").Inc() + } logger.Info("message observed", zap.String("txHash", observation.TxIDString()), diff --git a/node/pkg/watchers/cosmwasm/watcher.go b/node/pkg/watchers/cosmwasm/watcher.go index 0ec30b8b7b..ef799ce41f 100644 --- a/node/pkg/watchers/cosmwasm/watcher.go +++ b/node/pkg/watchers/cosmwasm/watcher.go @@ -12,6 +12,7 @@ import ( "github.com/certusone/wormhole/node/pkg/p2p" gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1" + "github.com/certusone/wormhole/node/pkg/watchers" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus" @@ -303,6 +304,7 @@ func (e *Watcher) Run(ctx context.Context) error { msg.IsReobservation = true e.msgC <- msg messagesConfirmed.WithLabelValues(networkName).Inc() + watchers.ReobservationsByChain.WithLabelValues(networkName, "std").Inc() } } } diff --git a/node/pkg/watchers/evm/watcher.go b/node/pkg/watchers/evm/watcher.go index 141a117376..124e103bea 100644 --- a/node/pkg/watchers/evm/watcher.go +++ b/node/pkg/watchers/evm/watcher.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/certusone/wormhole/node/pkg/watchers" "github.com/certusone/wormhole/node/pkg/watchers/evm/connectors" "github.com/certusone/wormhole/node/pkg/watchers/evm/connectors/ethabi" "github.com/certusone/wormhole/node/pkg/watchers/interfaces" @@ -620,8 +621,6 @@ func (w *Watcher) Run(parentCtx context.Context) error { zap.String("txHash", txHash.String()), zap.Error(pubErr), ) - } else { - ethMessagesConfirmed.WithLabelValues(w.networkName).Inc() } } } @@ -812,8 +811,6 @@ func (w *Watcher) postMessage( zap.String("txHash", msg.TxIDString()), zap.Error(pubErr), ) - } else { - ethMessagesConfirmed.WithLabelValues(w.networkName).Inc() } return } @@ -888,6 +885,10 @@ func (w *Watcher) verifyAndPublish( } w.msgC <- msg + ethMessagesConfirmed.WithLabelValues(w.networkName).Inc() + if msg.IsReobservation { + watchers.ReobservationsByChain.WithLabelValues(w.chainID.String(), "std").Inc() + } return nil } diff --git a/node/pkg/watchers/ibc/watcher.go b/node/pkg/watchers/ibc/watcher.go index 0292242751..2e9c881236 100644 --- a/node/pkg/watchers/ibc/watcher.go +++ b/node/pkg/watchers/ibc/watcher.go @@ -18,6 +18,7 @@ import ( "github.com/certusone/wormhole/node/pkg/p2p" "github.com/certusone/wormhole/node/pkg/readiness" "github.com/certusone/wormhole/node/pkg/supervisor" + "github.com/certusone/wormhole/node/pkg/watchers" "github.com/certusone/wormhole/node/pkg/watchers/cosmwasm" "github.com/wormhole-foundation/wormhole/sdk/vaa" @@ -673,6 +674,9 @@ func (w *Watcher) processIbcReceivePublishEvent(evt *ibcReceivePublishEvent, obs ce.msgC <- evt.Msg messagesConfirmed.WithLabelValues(ce.chainName).Inc() + if evt.Msg.IsReobservation { + watchers.ReobservationsByChain.WithLabelValues(evt.Msg.EmitterChain.String(), "std").Inc() + } return nil } diff --git a/node/pkg/watchers/near/tx_processing.go b/node/pkg/watchers/near/tx_processing.go index 5a8f272e69..7286e86144 100644 --- a/node/pkg/watchers/near/tx_processing.go +++ b/node/pkg/watchers/near/tx_processing.go @@ -12,6 +12,7 @@ import ( "time" "github.com/certusone/wormhole/node/pkg/common" + "github.com/certusone/wormhole/node/pkg/watchers" "github.com/certusone/wormhole/node/pkg/watchers/near/nearapi" eth_common "github.com/ethereum/go-ethereum/common" "github.com/mr-tron/base58" @@ -246,6 +247,10 @@ func (e *Watcher) processWormholeLog(logger *zap.Logger, _ context.Context, job IsReobservation: job.isReobservation, } + if job.isReobservation { + watchers.ReobservationsByChain.WithLabelValues("near", "std").Inc() + } + // tell everyone about it job.hasWormholeMsg = true diff --git a/node/pkg/watchers/solana/client.go b/node/pkg/watchers/solana/client.go index f2e560bced..88a23b1c9c 100644 --- a/node/pkg/watchers/solana/client.go +++ b/node/pkg/watchers/solana/client.go @@ -18,6 +18,7 @@ import ( "github.com/certusone/wormhole/node/pkg/query" "github.com/certusone/wormhole/node/pkg/readiness" "github.com/certusone/wormhole/node/pkg/supervisor" + "github.com/certusone/wormhole/node/pkg/watchers" eth_common "github.com/ethereum/go-ethereum/common" "github.com/gagliardetto/solana-go" lookup "github.com/gagliardetto/solana-go/programs/address-lookup-table" @@ -1062,6 +1063,9 @@ func (s *SolanaWatcher) processMessageAccount(logger *zap.Logger, data []byte, a } solanaMessagesConfirmed.WithLabelValues(s.networkName).Inc() + if isReobservation { + watchers.ReobservationsByChain.WithLabelValues(s.chainID.String(), "std").Inc() + } if logger.Level().Enabled(s.msgObservedLogLevel) { logger.Log(s.msgObservedLogLevel, "message observed", diff --git a/node/pkg/watchers/solana/shim.go b/node/pkg/watchers/solana/shim.go index 8917aeacf8..4b452a4d32 100644 --- a/node/pkg/watchers/solana/shim.go +++ b/node/pkg/watchers/solana/shim.go @@ -34,6 +34,7 @@ import ( "time" "github.com/certusone/wormhole/node/pkg/common" + "github.com/certusone/wormhole/node/pkg/watchers" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" "github.com/near/borsh-go" @@ -365,6 +366,9 @@ func (s *SolanaWatcher) shimProcessRest( } solanaMessagesConfirmed.WithLabelValues(s.networkName).Inc() + if isReobservation { + watchers.ReobservationsByChain.WithLabelValues(s.chainID.String(), "shim").Inc() + } if logger.Level().Enabled(s.msgObservedLogLevel) { logger.Log(s.msgObservedLogLevel, "message observed from shim", diff --git a/node/pkg/watchers/sui/watcher.go b/node/pkg/watchers/sui/watcher.go index 8327092ebc..3f54d179bf 100644 --- a/node/pkg/watchers/sui/watcher.go +++ b/node/pkg/watchers/sui/watcher.go @@ -17,6 +17,7 @@ import ( gossipv1 "github.com/certusone/wormhole/node/pkg/proto/gossip/v1" "github.com/certusone/wormhole/node/pkg/readiness" "github.com/certusone/wormhole/node/pkg/supervisor" + "github.com/certusone/wormhole/node/pkg/watchers" eth_common "github.com/ethereum/go-ethereum/common" "github.com/prometheus/client_golang/prometheus" @@ -274,6 +275,9 @@ func (e *Watcher) inspectBody(logger *zap.Logger, body SuiResult, isReobservatio } suiMessagesConfirmed.Inc() + if isReobservation { + watchers.ReobservationsByChain.WithLabelValues("sui", "std").Inc() + } logger.Info("message observed", zap.String("txHash", observation.TxIDString()), diff --git a/node/pkg/watchers/watchers.go b/node/pkg/watchers/watchers.go index 9bdc679ea0..21a10e9d5f 100644 --- a/node/pkg/watchers/watchers.go +++ b/node/pkg/watchers/watchers.go @@ -6,6 +6,8 @@ import ( "github.com/certusone/wormhole/node/pkg/query" "github.com/certusone/wormhole/node/pkg/supervisor" "github.com/certusone/wormhole/node/pkg/watchers/interfaces" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/wormhole-foundation/wormhole/sdk/vaa" ) @@ -27,3 +29,11 @@ type WatcherConfig interface { env common.Environment, ) (interfaces.L1Finalizer, supervisor.Runnable, interfaces.Reobserver, error) } + +var ( + ReobservationsByChain = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "wormhole_reobservations_by_chain", + Help: "Total number of reobservations completed by chain and observation type", + }, []string{"chain", "type"}) +)