diff options
Diffstat (limited to '')
| -rw-r--r-- | modules/by-name/mo/monitoring/components/prometheus.nix | 68 | ||||
| -rw-r--r-- | tests/by-name/mo/monitoring-federation/test.nix | 109 |
2 files changed, 176 insertions, 1 deletions
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix index 32477b6..ca947d3 100644 --- a/modules/by-name/mo/monitoring/components/prometheus.nix +++ b/modules/by-name/mo/monitoring/components/prometheus.nix @@ -21,6 +21,35 @@ in { default = 3001; }; + remoteWriteReceiver = lib.mkOption { + example = "https://prometheus.server2.vhack.eu"; + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + If not null, this prometheus server will be exposed via a reverse proxy and allow + other prometheus servers to write metrics to it. + ''; + }; + remoteWriteTo = lib.mkOption { + example = "https://prometheus.server3.vhack.eu"; + type = lib.types.nullOr (lib.types.submodule { + options = { + port = lib.mkOption { + type = lib.types.nullOr lib.types.port; + default = null; + }; + url = lib.mkOption { + type = lib.types.str; + }; + }; + }); + default = null; + description = '' + If not null, this prometheus server will run in agent mode, writing all recived + metrics to the specified remote prometheus server. + ''; + }; + sources = lib.mkOption { type = lib.types.listOf (lib.types.submodule { options = { @@ -29,6 +58,7 @@ in { }; target = lib.mkOption { type = lib.types.str; + description = "The target url to scrape from."; }; extra = lib.mkOption { type = lib.types.attrsOf lib.types.anything; @@ -67,6 +97,20 @@ in { ]; }; + services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) { + virtualHosts.${cfg.remoteWriteReceiver} = { + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write"; + + recommendedProxySettings = true; + proxyWebsockets = true; + }; + + enableACME = true; + forceSSL = true; + }; + }; + users = { groups = { node-exporter.gid = config.vhack.constants.ids.gids.node-exporter; @@ -88,6 +132,28 @@ in { globalConfig = { scrape_interval = "15s"; }; + remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [ + { + url = let + port = + if (cfg.remoteWriteTo.port != null) + then ":${toString cfg.remoteWriteTo.port}" + else ""; + in "${cfg.remoteWriteTo.url}${port}"; + # TODO: Needs RFC42 for prometheus <2026-07-19> + # protobuf_message = "io.prometheus.write.v2.Request"; + } + ]; + + enableAgentMode = cfg.remoteWriteTo != null; + extraFlags = lib.mkMerge [ + ( + lib.mkIf (cfg.remoteWriteReceiver != null) [ + "--web.enable-remote-write-receiver" + # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request" + ] + ) + ]; scrapeConfigs = builtins.map (attr: nixLib.warnMerge { @@ -99,7 +165,7 @@ in { } ]; } - attr.extra "prometheus scrapeConfig") + attr.extra "prometheus scrapeConfig '${attr.name}'") cfg.sources; # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # job_name = "redis"; diff --git a/tests/by-name/mo/monitoring-federation/test.nix b/tests/by-name/mo/monitoring-federation/test.nix new file mode 100644 index 0000000..a64577d --- /dev/null +++ b/tests/by-name/mo/monitoring-federation/test.nix @@ -0,0 +1,109 @@ +{ + extraModules, + pkgs, + vhack, + ... +}: let + prometheusMain = "prometheus.server2.server.org"; + remoteWriteTo = { + url = "https://${prometheusMain}"; + }; + promPort = 3001; +in + vhack.runTest { + name = "monitoring"; + + serverDomains = [ + {server2 = prometheusMain;} + ]; + + nodes = { + server2 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + environment.systemPackages = [ + pkgs.curl + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + remoteWriteReceiver = prometheusMain; + }; + }; + }; + }; + server3 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + inherit remoteWriteTo; + }; + }; + }; + }; + server4 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + inherit remoteWriteTo; + }; + }; + }; + }; + }; + + services = [ + {server2 = "prometheus.service";} + {server3 = "prometheus.service";} + {server4 = "prometheus.service";} + ]; + + testScript = {...}: + # Python + '' + import json, time + + # Give the Prometheus servers some time to generate metrics and sync with the main + # one. + time.sleep(1) + + metrics = server2.succeed("${pkgs.writeShellScript "query-metrics" '' + curl --silent \ + http://127.0.0.1:${toString promPort}/api/v1/query?query="go_gc_cleanups_queued_cleanups_total" + ''}") + + metrics = json.loads(metrics) + + hosts = [] + for result in metrics["data"]["result"]: + server2.log(json.dumps(result,indent=4)) + hosts.append(result["metric"]["hostname"]) + + hosts.sort() + + assert hosts == ["server2", "server3", "server4"], f"Not all hosts used the remote write (got {hosts})" + ''; + } |