From 15d214e2372308fa1d12b576a675c9e2cbf6cde1 Mon Sep 17 00:00:00 2001 From: Ellie Huxtable Date: Thu, 16 Nov 2023 23:18:13 +0000 Subject: feat: add metrics server and http metrics (#1394) * feat: add metrics server and http metrics * setup metrics * update default config * fix tests --- atuin-server/src/lib.rs | 27 ++++++++++++++++++++++- atuin-server/src/metrics.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++ atuin-server/src/router.rs | 4 +++- atuin-server/src/settings.rs | 21 ++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 atuin-server/src/metrics.rs (limited to 'atuin-server/src') diff --git a/atuin-server/src/lib.rs b/atuin-server/src/lib.rs index 6778b099..2d2a9c78 100644 --- a/atuin-server/src/lib.rs +++ b/atuin-server/src/lib.rs @@ -3,16 +3,20 @@ use std::{future::Future, net::TcpListener}; use atuin_server_database::Database; +use axum::Router; use axum::Server; use eyre::{Context, Result}; mod handlers; +mod metrics; mod router; -mod settings; mod utils; pub use settings::example_config; pub use settings::Settings; + +pub mod settings; + use tokio::signal; #[cfg(target_family = "unix")] @@ -70,3 +74,24 @@ pub async fn launch_with_listener( Ok(()) } + +// The separate listener means it's much easier to ensure metrics are not accidentally exposed to +// the public. +pub async fn launch_metrics_server(host: String, port: u16) -> Result<()> { + let listener = TcpListener::bind((host, port)).context("failed to bind metrics tcp")?; + + let recorder_handle = metrics::setup_metrics_recorder(); + + let router = Router::new().route( + "/metrics", + axum::routing::get(move || std::future::ready(recorder_handle.render())), + ); + + Server::from_tcp(listener) + .context("could not launch server")? + .serve(router.into_make_service()) + .with_graceful_shutdown(shutdown_signal()) + .await?; + + Ok(()) +} diff --git a/atuin-server/src/metrics.rs b/atuin-server/src/metrics.rs new file mode 100644 index 00000000..2e3e6894 --- /dev/null +++ b/atuin-server/src/metrics.rs @@ -0,0 +1,52 @@ +use std::time::Instant; + +use axum::{extract::MatchedPath, http::Request, middleware::Next, response::IntoResponse}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; + +pub fn setup_metrics_recorder() -> PrometheusHandle { + const EXPONENTIAL_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + PrometheusBuilder::new() + .set_buckets_for_metric( + Matcher::Full("http_requests_duration_seconds".to_string()), + EXPONENTIAL_SECONDS, + ) + .unwrap() + .install_recorder() + .unwrap() +} + +/// Middleware to record some common HTTP metrics +/// Generic over B to allow for arbitrary body types (eg Vec, Streams, a deserialized thing, etc) +/// Someday tower-http might provide a metrics middleware: https://github.com/tower-rs/tower-http/issues/57 +pub async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { + let start = Instant::now(); + + let path = if let Some(matched_path) = req.extensions().get::() { + matched_path.as_str().to_owned() + } else { + req.uri().path().to_owned() + }; + + let method = req.method().clone(); + + // Run the rest of the request handling first, so we can measure it and get response + // codes. + let response = next.run(req).await; + + let latency = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + let labels = [ + ("method", method.to_string()), + ("path", path), + ("status", status), + ]; + + metrics::increment_counter!("http_requests_total", &labels); + metrics::histogram!("http_requests_duration_seconds", latency, &labels); + + response +} diff --git a/atuin-server/src/router.rs b/atuin-server/src/router.rs index e1220e56..90e726d3 100644 --- a/atuin-server/src/router.rs +++ b/atuin-server/src/router.rs @@ -16,6 +16,7 @@ use tower_http::trace::TraceLayer; use super::handlers; use crate::{ handlers::{ErrorResponseStatus, RespExt}, + metrics, settings::Settings, }; use atuin_server_database::{models::User, Database, DbError}; @@ -124,6 +125,7 @@ pub fn router(database: DB, settings: Settings) -> R .layer( ServiceBuilder::new() .layer(axum::middleware::from_fn(clacks_overhead)) - .layer(TraceLayer::new_for_http()), + .layer(TraceLayer::new_for_http()) + .layer(axum::middleware::from_fn(metrics::track_metrics)), ) } diff --git a/atuin-server/src/settings.rs b/atuin-server/src/settings.rs index 744f4ec2..d6f1867c 100644 --- a/atuin-server/src/settings.rs +++ b/atuin-server/src/settings.rs @@ -7,6 +7,23 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; static EXAMPLE_CONFIG: &str = include_str!("../server.toml"); +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Metrics { + pub enable: bool, + pub host: String, + pub port: u16, +} + +impl Default for Metrics { + fn default() -> Self { + Self { + enable: false, + host: String::from("127.0.0.1"), + port: 9001, + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Settings { pub host: String, @@ -18,6 +35,7 @@ pub struct Settings { pub page_size: i64, pub register_webhook_url: Option, pub register_webhook_username: String, + pub metrics: Metrics, #[serde(flatten)] pub db_settings: DbSettings, @@ -46,6 +64,9 @@ impl Settings { .set_default("path", "")? .set_default("register_webhook_username", "")? .set_default("page_size", 1100)? + .set_default("metrics.enable", false)? + .set_default("metrics.host", "127.0.0.1")? + .set_default("metrics.port", 9001)? .add_source( Environment::with_prefix("atuin") .prefix_separator("_") -- cgit v1.3.1