diff options
Diffstat (limited to '')
| -rw-r--r-- | modules/by-name/mo/monitoring/components/prometheus.nix | 120 |
1 files changed, 100 insertions, 20 deletions
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix index 468ece3..ca947d3 100644 --- a/modules/by-name/mo/monitoring/components/prometheus.nix +++ b/modules/by-name/mo/monitoring/components/prometheus.nix @@ -8,7 +8,6 @@ commonLabels = { hostname = config.networking.hostName; - inherit (config.vhack.monitoring.grafana) fqdn; }; prom_cfg = config.services.prometheus; @@ -22,6 +21,35 @@ in { default = 3001; }; + remoteWriteReceiver = lib.mkOption { + example = "https://prometheus.server2.vhack.eu"; + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + If not null, this prometheus server will be exposed via a reverse proxy and allow + other prometheus servers to write metrics to it. + ''; + }; + remoteWriteTo = lib.mkOption { + example = "https://prometheus.server3.vhack.eu"; + type = lib.types.nullOr (lib.types.submodule { + options = { + port = lib.mkOption { + type = lib.types.nullOr lib.types.port; + default = null; + }; + url = lib.mkOption { + type = lib.types.str; + }; + }; + }); + default = null; + description = '' + If not null, this prometheus server will run in agent mode, writing all recived + metrics to the specified remote prometheus server. + ''; + }; + sources = lib.mkOption { type = lib.types.listOf (lib.types.submodule { options = { @@ -30,6 +58,7 @@ in { }; target = lib.mkOption { type = lib.types.str; + description = "The target url to scrape from."; }; extra = lib.mkOption { type = lib.types.attrsOf lib.types.anything; @@ -42,16 +71,22 @@ in { config = lib.mkIf cfg.enable { vhack = { nginx.enable = true; + + persist.directories = [ + { + directory = "/var/lib/prometheus2"; + user = "prometheus"; + group = "prometheus"; + mode = "0700"; + } + ]; + monitoring.prometheus.sources = [ { name = "node"; target = "127.0.0.1:${toString prom_cfg.exporters.node.port}"; } { - name = "smartctl"; - target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}"; - } - { name = "prometheus_internal"; target = "127.0.0.1:${toString prom_cfg.port}"; } @@ -59,18 +94,36 @@ in { name = "systemd"; target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}"; } - { - name = "netdata"; - target = "127.0.0.1:19999"; - extra = { - metrics_path = "/api/v1/allmetrics"; - params.format = ["prometheus"]; - honor_labels = true; - }; - } ]; }; + services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) { + virtualHosts.${cfg.remoteWriteReceiver} = { + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write"; + + recommendedProxySettings = true; + proxyWebsockets = true; + }; + + enableACME = true; + forceSSL = true; + }; + }; + + users = { + groups = { + node-exporter.gid = config.vhack.constants.ids.gids.node-exporter; + }; + users = { + node-exporter = { + isSystemUser = true; + group = "node-exporter"; + uid = config.vhack.constants.ids.uids.node-exporter; + }; + }; + }; + services = { prometheus = { enable = true; @@ -79,6 +132,28 @@ in { globalConfig = { scrape_interval = "15s"; }; + remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [ + { + url = let + port = + if (cfg.remoteWriteTo.port != null) + then ":${toString cfg.remoteWriteTo.port}" + else ""; + in "${cfg.remoteWriteTo.url}${port}"; + # TODO: Needs RFC42 for prometheus <2026-07-19> + # protobuf_message = "io.prometheus.write.v2.Request"; + } + ]; + + enableAgentMode = cfg.remoteWriteTo != null; + extraFlags = lib.mkMerge [ + ( + lib.mkIf (cfg.remoteWriteReceiver != null) [ + "--web.enable-remote-write-receiver" + # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request" + ] + ) + ]; scrapeConfigs = builtins.map (attr: nixLib.warnMerge { @@ -90,7 +165,7 @@ in { } ]; } - attr.extra "prometheus scrapeConfig") + attr.extra "prometheus scrapeConfig '${attr.name}'") cfg.sources; # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # job_name = "redis"; @@ -159,11 +234,6 @@ in { port = 9112; listenAddress = "127.0.0.1"; }; - smartctl = { - enable = true; - port = 9115; - listenAddress = "127.0.0.1"; - }; # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # enable = true; # port = 9119; @@ -184,6 +254,16 @@ in { enable = true; port = 9116; listenAddress = "127.0.0.1"; + extraFlags = [ + "--log.level=info" + "--web.disable-exporter-metrics" + # "--systemd.collector.enable-resolved" + "--systemd.collector.enable-restart-count" + # "--systemd.collector.enable-file-descriptor-size" + "--systemd.collector.enable-ip-accounting" + "--systemd.collector.unit-include=\".+\"" + "--systemd.collector.unit-exclude=\".+\\\\.(device)\"" + ]; }; }; }; |