use std::{sync::LazyLock, time::Duration}; pub(crate) mod networking_metrics; pub(crate) mod tokio_runtime_metrics; pub(crate) mod tokio_task_metrics; const MONITOR_SAMPLE_DURATION: Duration = Duration::from_secs(10); pub static MPC_NUM_TRIPLES_GENERATED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_triples_generated", "Number of triples generated (including both owned and not owned)" ) .unwrap() }); pub static MPC_NUM_BAD_PEER_PRESIGN_REQUESTS: LazyLock = LazyLock::new(|| { prometheus::register_int_counter_vec!( "mpc_num_bad_peer_presign_requests", "Presignature requests from a peer whose participant-set size did not \ match the domain's reconstruction threshold", &["domain_id"] ) .unwrap() }); pub static MPC_TRIPLES_GENERATION_TIME_ELAPSED: LazyLock = LazyLock::new(|| { prometheus::register_histogram!( "near_mpc_triples_generation_time_elapsed", "Time taken to generate a batch of triples", // Batch generation takes tens of seconds, up to the configured // timeout; the default buckets top out at 10s. vec![5.0, 10.0, 20.0, 30.0, 45.0, 60.0, 90.0, 120.0, 180.0, 240.0], ) .unwrap() }); pub static MPC_PRE_SIGNATURE_TIME_ELAPSED: LazyLock = LazyLock::new(|| { prometheus::register_histogram!( "near_mpc_pre_signature_time_elapsed", "Time taken to generate a pre signature", ) .unwrap() }); pub static MPC_SIGNATURE_TIME_ELAPSED: LazyLock = LazyLock::new(|| { prometheus::register_histogram!( "near_mpc_signature_time_elapsed", "Time taken to generate a signature", ) .unwrap() }); pub static MPC_CKD_TIME_ELAPSED: LazyLock = LazyLock::new(|| { prometheus::register_histogram!( "near_mpc_ckd_time_elapsed", "Time taken to generate a confidential key", ) .unwrap() }); pub static MPC_OWNED_NUM_TRIPLES_AVAILABLE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_triples_available", "Number of triples generated that we own, and not yet used" ) .unwrap() }); pub static MPC_OWNED_NUM_TRIPLES_ONLINE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_triples_online", "Number of triples generated that we own, and not yet used, for which the participant set is confirmed alive" ) .unwrap() }); pub static MPC_OWNED_NUM_TRIPLES_WITH_OFFLINE_PARTICIPANT: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_triples_with_offline_participant", "Number of triples generated that we own, and not yet used, for which some participant is offline", ) .unwrap() }); pub static MPC_OWNED_NUM_PRESIGNATURES_AVAILABLE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_presignatures_available", "Number of presignatures generated that we own, and not yet used" ) .unwrap() }); pub static MPC_OWNED_NUM_PRESIGNATURES_ONLINE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_presignatures_online", "Number of presignatures generated that we own, and not yet used, for which the participant set is confirmed alive" ) .unwrap() }); pub static MPC_OWNED_NUM_PRESIGNATURES_WITH_OFFLINE_PARTICIPANT: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_owned_num_presignatures_with_offline_participant", "Number of presignatures generated that we own, and not yet used, for which some participant is offline", ) .unwrap() }); pub static MPC_INDEXER_NUM_RECEIPT_EXECUTION_OUTCOMES: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_indexer_num_receipt_execution_outcomes", "Number of receipt execution outcomes processed by the near indexer" ) .unwrap() }); pub static MPC_NUM_SIGN_REQUESTS_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_signature_requests_indexed", "Number of signatures requests seen by the indexer" ) .unwrap() }); pub static MPC_NUM_CKD_REQUESTS_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_ckd_requests_indexed", "Number of ckd requests seen by the indexer" ) .unwrap() }); pub static MPC_NUM_VERIFY_FOREIGN_TX_REQUESTS_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_verify_foreign_tx_requests_indexed", "Number of verify foreign tx requests seen by the indexer" ) .unwrap() }); pub static MPC_NUM_SIGN_RESPONSES_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_signature_responses_indexed", "Number of signatures responses seen by the indexer" ) .unwrap() }); pub static MPC_NUM_CKD_RESPONSES_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_ckd_responses_indexed", "Number of ckd responses seen by the indexer" ) .unwrap() }); pub static MPC_NUM_VERIFY_FOREIGN_TX_RESPONSES_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_verify_foreign_tx_responses_indexed", "Number of verify foreign tx responses seen by the indexer" ) .unwrap() }); pub static MPC_NUM_TIMEOUTS_INDEXED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_fail_on_timeout_indexed", "Number of calls to fail_on_timeout seen by the indexer." ) .unwrap() }); pub static MPC_NUM_SIGNATURE_COMPUTATIONS_LED: LazyLock = LazyLock::new( || { prometheus::register_int_counter_vec!( "mpc_num_signature_computations_led", "Number of finished signature computation attempts that this node led. Attempts interrupted before completion (e.g. node restart) are not counted.", &["result"], ) .unwrap() }, ); pub static MPC_NUM_CKD_COMPUTATIONS_LED: LazyLock = LazyLock::new( || { prometheus::register_int_counter_vec!( "mpc_num_ckd_computations_led", "Number of finished ckd computation attempts that this node led. Attempts interrupted before completion (e.g. node restart) are not counted.", &["result"], ) .unwrap() }, ); pub static MPC_NUM_VERIFY_FOREIGN_TX_COMPUTATIONS_LED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter_vec!( "mpc_num_verify_foreign_tx_computations_led", "Number of finished verify foreign tx computation attempts that this node led. Attempts interrupted before completion (e.g. node restart) are not counted.", &["result"], ) .unwrap() }); pub static MPC_NUM_PASSIVE_SIGN_REQUESTS_RECEIVED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_passive_signature_requests_received", "Number of passive signature requests received from mpc peers" ) .unwrap() }); pub static MPC_NUM_PASSIVE_CKD_REQUESTS_RECEIVED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_passive_ckd_requests_received", "Number of passive ckd requests received from mpc peers" ) .unwrap() }); pub static MPC_NUM_PASSIVE_SIGN_REQUESTS_LOOKUP_SUCCEEDED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_passive_signature_requests_lookup_succeeded", "Number of passive signature requests successfully looked up in local DB" ) .unwrap() }); pub static MPC_NUM_PASSIVE_CKD_REQUESTS_LOOKUP_SUCCEEDED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "mpc_num_passive_ckd_requests_lookup_succeeded", "Number of passive ckd requests successfully looked up in local DB" ) .unwrap() }); pub static MPC_OUTGOING_TRANSACTION_OUTCOMES: LazyLock = LazyLock::new(|| { prometheus::register_int_counter_vec!( "mpc_outgoing_transaction_outcomes", "Number of transactions sent by this node, by type and outcome", &["type", "outcome"], ) .unwrap() }); pub static MPC_INDEXER_LATEST_BLOCK_HEIGHT: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_indexer_latest_block_height", "Latest block height processed by the near indexer" ) .unwrap() }); pub static MPC_INDEXER_LATEST_BLOCK_TIMESTAMP_SECONDS: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_indexer_latest_block_timestamp_seconds", "Unix time of the latest block processed by the near indexer" ) .unwrap() }); pub static MPC_CURRENT_EPOCH_ID: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_current_epoch_id", "Epoch id of the keyset the contract currently holds, as last observed by this node" ) .unwrap() }); pub static MPC_LAST_BACKUP_SERVED_EPOCH: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_last_backup_served_epoch", "Epoch id of the most recent keyset successfully served to the registered backup service" ) .unwrap() }); pub static MPC_LAST_BACKUP_SERVED_TIMESTAMP_SECONDS: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_last_backup_served_timestamp_seconds", "Unix time at which keyshares were most recently served to the registered backup \ service" ) .unwrap() }); pub static MPC_OWN_IMAGE_HASH_ALLOWED: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_own_image_hash_allowed", "Whether the Docker image hash this node is running is in the contract's \ allowlist (1) or not (0)" ) .unwrap() }); pub static MPC_OWN_IMAGE_HASH_IS_MOST_RECENT: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_own_image_hash_is_most_recent", "Whether the Docker image hash this node is running is the most recent \ hash in the contract's allowlist (1) or not (0)" ) .unwrap() }); pub static MPC_OWN_IMAGE_HASH_EXPIRY_TIMESTAMP_SECONDS: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_own_image_hash_expiry_timestamp_seconds", "Unix time at which the Docker image hash this node is running is evicted \ from the contract's allowlist. -1 when the hash never expires (newest \ entry) or the contract does not report expiry; 0 when the hash is already \ evicted. Compare against mpc_indexer_latest_block_timestamp_seconds for \ the remaining time" ) .unwrap() }); pub static MPC_ACCESS_KEY_NONCE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge!( "mpc_access_key_nonce", "Latest observed nonce among transactions submitted using the node's access key for its near account", ) .unwrap() }); pub static MPC_CURRENT_JOB_STATE: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge_vec!( "mpc_current_job_state", "Current state of the top-level MPC job", &["state"], ) .unwrap() }); pub static SIGN_REQUEST_CHANNEL_FAILED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "sign_request_channel_failed", "failed to send on channel in sign_request_channel", ) .unwrap() }); pub static CKD_REQUEST_CHANNEL_FAILED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "ckd_request_channel_failed", "failed to send on channel in ckd_request_channel", ) .unwrap() }); pub static VERIFY_FOREIGN_TX_REQUEST_CHANNEL_FAILED: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "verify_foreign_tx_request_channel_failed", "failed to send on channel in verify_foreign_tx_request_channel", ) .unwrap() }); pub static VERIFY_TEE_REQUESTS_SENT: LazyLock = LazyLock::new(|| { prometheus::register_int_counter!( "verify_tee_requests_sent", "failed to send on channel in sign_request_channel", ) .unwrap() }); pub static PEERS_INDEXER_HEIGHTS: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge_vec!( "mpc_peers_indexer_block_heights", "Latest known block height of peers", &["participant"] ) .unwrap() }); pub static MPC_BUILD_INFO: LazyLock = LazyLock::new(|| { prometheus::register_int_gauge_vec!( "mpc_node_build_info", "Metric whose labels indicate node’s version", &["release", "build_time", "commit", "rustc_version"], ) .unwrap() }); /// Initialize the build info metric with current version information pub fn init_build_info_metric() { // Use compile-time constants from built crate let version = crate::built_info::PKG_VERSION; let build_time = crate::built_info::BUILT_TIME_UTC; let commit = crate::built_info::GIT_COMMIT_HASH_SHORT.unwrap_or("unknown"); let rustc_version = crate::built_info::RUSTC_VERSION; MPC_BUILD_INFO .with_label_values(&[version, build_time, commit, rustc_version]) .set(1); } pub static PARTICIPANT_TOTAL_TIMES_SEEN_IN_FAILED_SIGNATURE_COMPUTATION_LEADER: LazyLock< prometheus::IntCounterVec, > = LazyLock::new(|| { prometheus::register_int_counter_vec!( "participant_total_times_seen_in_failed_signature_computation_leader", "Number of times each participant id was seen in a failed signature computation that was led by us", &["participant_id"], ) .unwrap() }); pub static PARTICIPANT_TOTAL_TIMES_SEEN_IN_FAILED_SIGNATURE_COMPUTATION_FOLLOWER: LazyLock< prometheus::IntCounterVec, > = LazyLock::new(|| { prometheus::register_int_counter_vec!( "participant_total_times_seen_in_failed_signature_computation_follower", "Number of times each participant id was seen in a failed signature computation that was followed by us", &["participant_id"], ) .unwrap() }); pub const MPC_NUM_COMPUTATIONS_LED_TOTAL_LABEL: &str = "total"; pub const MPC_NUM_COMPUTATIONS_LED_SUCCEEDED_LABEL: &str = "succeeded"; pub const MPC_NUM_COMPUTATIONS_LED_FAILED_LABEL: &str = "failed"; pub const MPC_NUM_COMPUTATIONS_LED_DEADLINE_EXCEEDED_LABEL: &str = "deadline_exceeded"; pub static MPC_TEE_ATTESTATION_ATTEMPTS_TOTAL: LazyLock = LazyLock::new(|| { prometheus::register_int_counter_vec!( "mpc_tee_attestation_attempts_total", "Total number of completed TEE attestation generation attempts", &["outcome"], ) .unwrap() }); pub const MPC_TEE_ATTESTATION_OUTCOME_SUCCESS: &str = "success"; pub const MPC_TEE_ATTESTATION_OUTCOME_FAILURE: &str = "failure";