From 75d3d44fb35fee5230a0c98f493437ef2ae22038 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 14 Feb 2020 13:38:33 -0800 Subject: [PATCH] Metrics MVP: add two metrics and export them to Prometheus. Co-authored-by: Deirdre Connolly --- Cargo.lock | 298 +++++++++++++++++++++++++++++- zebra-network/Cargo.toml | 2 + zebra-network/src/peer_set/set.rs | 9 + zebrad/Cargo.toml | 3 + zebrad/src/application.rs | 5 +- zebrad/src/components.rs | 1 + zebrad/src/components/metrics.rs | 48 +++++ zebrad/src/components/tracing.rs | 9 +- zebrad/src/config.rs | 21 ++- 9 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 zebrad/src/components/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 648d7a14..542bf34c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "arc-swap" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841" + [[package]] name = "arc-swap" version = "0.4.4" @@ -268,6 +274,41 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "crossbeam-epoch" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +dependencies = [ + "autocfg 0.1.7", + "cfg-if", + "crossbeam-utils 0.7.0", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +dependencies = [ + "autocfg 0.1.7", + "cfg-if", + "lazy_static", +] + [[package]] name = "curve25519-dalek" version = "2.0.0" @@ -325,6 +366,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dtoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" + [[package]] name = "ed25519-zebra" version = "0.2.0" @@ -531,6 +578,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "hdrhistogram" +version = "6.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "hermit-abi" version = "0.1.6" @@ -603,6 +660,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "im" +version = "12.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de38d1511a0ce7677538acb1e31b5df605147c458e061b2cdb89858afb1cd182" +dependencies = [ + "rustc_version", + "sized-chunks", + "typenum", +] + [[package]] name = "indexmap" version = "1.3.2" @@ -659,6 +727,21 @@ version = "0.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.8" @@ -689,6 +772,118 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +[[package]] +name = "memoffset" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "metrics" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b70227ece8711a1aa2f99655efd795d0cff297a5b9fe39645a93aacf6ad39d" +dependencies = [ + "metrics-core", +] + +[[package]] +name = "metrics-core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c064b3a1ff41f4bf6c91185c8a0caeccf8a8a27e9d0f92cc54cf3dbec812f48" + +[[package]] +name = "metrics-exporter-http" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14017d204ae062dc5c68a321e3dbdcd9b30181305cb6b067932f7f03f754e27" +dependencies = [ + "hyper", + "log", + "metrics-core", +] + +[[package]] +name = "metrics-exporter-log" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3fc63816bd5f8bde5eb31ce471f9633adc69ba1c55b44191b4d5fc7e263e8ab" +dependencies = [ + "log", + "metrics-core", + "tokio", +] + +[[package]] +name = "metrics-observer-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe930460a6c336b8f873dcfb28da3f805fd0dbadbea7beaf3042c7fb1d9fcd3" +dependencies = [ + "hdrhistogram", + "metrics-core", + "metrics-util", + "serde_json", +] + +[[package]] +name = "metrics-observer-prometheus" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9bb94f40e189c87cf70ef1c78815b949ab9d28fe76ebb81f15f79bd19a33d6" +dependencies = [ + "hdrhistogram", + "metrics-core", + "metrics-util", +] + +[[package]] +name = "metrics-observer-yaml" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f66811013592560efc75d75a92d6e2f415a11b52f085e51d9fb4d1edec6335" +dependencies = [ + "hdrhistogram", + "metrics-core", + "metrics-util", + "serde_yaml", +] + +[[package]] +name = "metrics-runtime" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb3035626782c533953bcc1a349467543b7f0e9d7b0c92edc61ee2bda7033d6" +dependencies = [ + "arc-swap 0.3.11", + "crossbeam-utils 0.6.6", + "im", + "metrics", + "metrics-core", + "metrics-exporter-http", + "metrics-exporter-log", + "metrics-observer-json", + "metrics-observer-prometheus", + "metrics-observer-yaml", + "metrics-util", + "parking_lot", + "quanta", +] + +[[package]] +name = "metrics-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d11f8090a8886339f9468a04eeea0711e4cf27538b134014664308041307a1c5" +dependencies = [ + "crossbeam-epoch", + "serde", +] + [[package]] name = "mio" version = "0.6.21" @@ -792,6 +987,32 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core", + "rustc_version", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec", + "winapi 0.3.8", +] + [[package]] name = "pin-project" version = "0.4.8" @@ -896,6 +1117,16 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "quanta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7a1905379198075914bc93d32a5465c40474f90a078bb13439cb00c547bcc" +dependencies = [ + "libc", + "winapi 0.3.8", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1139,6 +1370,15 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + [[package]] name = "rusty-fork" version = "0.2.2" @@ -1151,6 +1391,18 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" + +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" + [[package]] name = "secrecy" version = "0.6.0" @@ -1197,6 +1449,29 @@ dependencies = [ "syn 1.0.13", ] +[[package]] +name = "serde_json" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + [[package]] name = "sha2" version = "0.8.1" @@ -1225,10 +1500,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" dependencies = [ - "arc-swap", + "arc-swap 0.4.4", "libc", ] +[[package]] +name = "sized-chunks" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3e7f23bad2d6694e0f46f5e470ec27eb07b8f3e8b309a4b0dc17501928b9f2" +dependencies = [ + "typenum", +] + [[package]] name = "slab" version = "0.4.2" @@ -1710,6 +1994,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zebra-chain" version = "0.1.0" @@ -1746,6 +2039,7 @@ dependencies = [ "futures", "hex", "indexmap", + "metrics", "pin-project", "proptest", "proptest-derive", @@ -1782,6 +2076,8 @@ dependencies = [ "futures", "gumdrop", "hyper", + "metrics", + "metrics-runtime", "once_cell", "rand 0.7.3", "serde", diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml index a3375c28..4dcbac5c 100644 --- a/zebra-network/Cargo.toml +++ b/zebra-network/Cargo.toml @@ -33,4 +33,6 @@ tracing-futures = "0.2" tower = "0.3" tower-load = "0.3" +metrics = "0.12" + zebra-chain = { path = "../zebra-chain" } diff --git a/zebra-network/src/peer_set/set.rs b/zebra-network/src/peer_set/set.rs index 6db759e6..eb0794ac 100644 --- a/zebra-network/src/peer_set/set.rs +++ b/zebra-network/src/peer_set/set.rs @@ -1,5 +1,6 @@ use std::{ collections::HashMap, + convert::TryInto, fmt::Debug, future::Future, marker::PhantomData, @@ -230,6 +231,14 @@ where num_ready = self.ready_services.len(), num_unready = self.unready_services.len(), ); + metrics::gauge!( + "pool.num_ready", + self.ready_services.len().try_into().unwrap() + ); + metrics::gauge!( + "pool.num_unready", + self.unready_services.len().try_into().unwrap() + ); loop { // Re-check that the pre-selected service is ready, in case diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index f97278fd..d411c83e 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -25,6 +25,9 @@ hyper = "0.13.2" tower = "0.3" +metrics-runtime = "0.13" +metrics = "0.12" + zebra-chain = { path = "../zebra-chain" } zebra-network = { path = "../zebra-network" } diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index 576cdc41..5291b9a2 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -82,11 +82,14 @@ impl Application for ZebradApp { /// beyond the default ones provided by the framework, this is the place /// to do so. fn register_components(&mut self, command: &Self::Cmd) -> Result<(), FrameworkError> { - use crate::components::{tokio::TokioComponent, tracing::TracingEndpoint}; + use crate::components::{ + metrics::MetricsEndpoint, tokio::TokioComponent, tracing::TracingEndpoint, + }; let mut components = self.framework_components(command)?; components.push(Box::new(TokioComponent::new()?)); components.push(Box::new(TracingEndpoint::new()?)); + components.push(Box::new(MetricsEndpoint::new()?)); self.state.components.register(components) } diff --git a/zebrad/src/components.rs b/zebrad/src/components.rs index a5c36e7d..59d7b759 100644 --- a/zebrad/src/components.rs +++ b/zebrad/src/components.rs @@ -1,2 +1,3 @@ +pub mod metrics; pub mod tokio; pub mod tracing; diff --git a/zebrad/src/components/metrics.rs b/zebrad/src/components/metrics.rs new file mode 100644 index 00000000..4c34bd83 --- /dev/null +++ b/zebrad/src/components/metrics.rs @@ -0,0 +1,48 @@ +//! An HTTP endpoint for metrics collection. + +use metrics_runtime::{exporters::HttpExporter, observers::PrometheusBuilder, Receiver}; + +use abscissa_core::{Component, FrameworkError}; + +use crate::{components::tokio::TokioComponent, prelude::*}; + +/// Abscissa component which runs a metrics endpoint. +#[derive(Debug, Component)] +#[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")] +pub struct MetricsEndpoint {} + +impl MetricsEndpoint { + /// Create the component. + pub fn new() -> Result { + Ok(Self {}) + } + + /// Do setup after receiving a tokio runtime. + pub fn init_tokio(&mut self, tokio_component: &TokioComponent) -> Result<(), FrameworkError> { + info!("Initializing metrics endpoint"); + + // XXX load metrics addr from config + let addr = "127.0.0.1:9999" + .parse() + .expect("Hardcoded address should be parseable"); + + // XXX do we need to hold on to the receiver? + let receiver = Receiver::builder() + .build() + .expect("Receiver config should be valid"); + // XXX ???? connect this ??? + let _sink = receiver.sink(); + + let endpoint = HttpExporter::new(receiver.controller(), PrometheusBuilder::new(), addr); + + tokio_component + .rt + .as_ref() + .expect("runtime should not be taken") + .spawn(endpoint.async_run()); + + metrics::set_boxed_recorder(Box::new(receiver)).expect("XXX FIXME ERROR CONVERSION"); + + Ok(()) + } +} diff --git a/zebrad/src/components/tracing.rs b/zebrad/src/components/tracing.rs index 260db884..91a0e83f 100644 --- a/zebrad/src/components/tracing.rs +++ b/zebrad/src/components/tracing.rs @@ -8,17 +8,10 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; /// Abscissa component which runs a tracing filter endpoint. -#[derive(Component)] +#[derive(Debug, Component)] #[component(inject = "init_tokio(zebrad::components::tokio::TokioComponent)")] pub struct TracingEndpoint {} -impl ::std::fmt::Debug for TracingEndpoint { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { - // Debug is required by Component, can't be derived as a Handle is not Debug - write!(f, "TracingEndpoint") - } -} - async fn read_filter(req: Request) -> Result { std::str::from_utf8( &hyper::body::to_bytes(req.into_body()) diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index 7f0fdaae..d57ef6af 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -4,6 +4,8 @@ //! application's configuration file and/or command-line options //! for specifying it. +use std::net::SocketAddr; + use serde::{Deserialize, Serialize}; use zebra_network::Config as NetworkSection; @@ -14,8 +16,10 @@ use zebra_network::Config as NetworkSection; pub struct ZebradConfig { /// Tracing configuration pub tracing: TracingSection, - /// Networking configuration, + /// Networking configuration pub network: NetworkSection, + /// Metrics configuration + pub metrics: MetricsSection, } /// Tracing configuration section. @@ -33,3 +37,18 @@ impl Default for TracingSection { } } } + +/// Metrics configuration section. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct MetricsSection { + pub endpoint_addr: SocketAddr, +} + +impl Default for MetricsSection { + fn default() -> Self { + Self { + endpoint_addr: "127.0.0.1:9999".parse().unwrap(), + } + } +}