{ lib, config, nixLib, ... }: let cfg = config.vhack.monitoring.prometheus; commonLabels = { hostname = config.networking.hostName; }; prom_cfg = config.services.prometheus; in { options.vhack.monitoring.prometheus = { enable = lib.mkEnableOption "prometheus"; port = lib.mkOption { type = lib.types.port; description = "Port where Prometheus listens to HTTP requests."; default = 3001; }; remoteWriteReceiver = lib.mkOption { example = "https://prometheus.server2.vhack.eu"; type = lib.types.nullOr lib.types.str; default = null; description = '' If not null, this prometheus server will be exposed via a reverse proxy and allow other prometheus servers to write metrics to it. ''; }; remoteWriteTo = lib.mkOption { example = "https://prometheus.server3.vhack.eu"; type = lib.types.nullOr (lib.types.submodule { options = { port = lib.mkOption { type = lib.types.nullOr lib.types.port; default = null; }; url = lib.mkOption { type = lib.types.str; }; }; }); default = null; description = '' If not null, this prometheus server will run in agent mode, writing all recived metrics to the specified remote prometheus server. ''; }; sources = lib.mkOption { type = lib.types.listOf (lib.types.submodule { options = { name = lib.mkOption { type = lib.types.str; }; target = lib.mkOption { type = lib.types.str; description = "The target url to scrape from."; }; extra = lib.mkOption { type = lib.types.attrsOf lib.types.anything; }; }; }); }; }; config = lib.mkIf cfg.enable { vhack = { nginx.enable = true; persist.directories = [ { directory = "/var/lib/prometheus2"; user = "prometheus"; group = "prometheus"; mode = "0700"; } ]; monitoring.prometheus.sources = [ { name = "node"; target = "127.0.0.1:${toString prom_cfg.exporters.node.port}"; } { name = "prometheus_internal"; target = "127.0.0.1:${toString prom_cfg.port}"; } { name = "systemd"; target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}"; } ]; }; services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) { virtualHosts.${cfg.remoteWriteReceiver} = { locations."/" = { proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write"; recommendedProxySettings = true; proxyWebsockets = true; }; enableACME = true; forceSSL = true; }; }; users = { groups = { node-exporter.gid = config.vhack.constants.ids.gids.node-exporter; }; users = { node-exporter = { isSystemUser = true; group = "node-exporter"; uid = config.vhack.constants.ids.uids.node-exporter; }; }; }; services = { prometheus = { enable = true; inherit (cfg) port; globalConfig = { scrape_interval = "15s"; }; remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [ { url = let port = if (cfg.remoteWriteTo.port != null) then ":${toString cfg.remoteWriteTo.port}" else ""; in "${cfg.remoteWriteTo.url}${port}"; # TODO: Needs RFC42 for prometheus <2026-07-19> # protobuf_message = "io.prometheus.write.v2.Request"; } ]; enableAgentMode = cfg.remoteWriteTo != null; extraFlags = lib.mkMerge [ ( lib.mkIf (cfg.remoteWriteReceiver != null) [ "--web.enable-remote-write-receiver" # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request" ] ) ]; scrapeConfigs = builtins.map (attr: nixLib.warnMerge { job_name = attr.name; static_configs = [ { targets = [attr.target]; labels = commonLabels; } ]; } attr.extra "prometheus scrapeConfig '${attr.name}'") cfg.sources; # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # job_name = "redis"; # static_configs = [ # { # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"]; # } # ]; # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) { # job_name = "openvpn"; # static_configs = [ # { # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"]; # } # ]; # }) # ++ (lib.optional config.services.dnsmasq.enable { # job_name = "dnsmasq"; # static_configs = [ # { # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"]; # labels = commonLabels; # } # ]; # }); exporters = { node = { enable = true; # https://github.com/prometheus/node_exporter#collectors enabledCollectors = [ "arp" "cpu" "cpufreq" "diskstats" "dmi" "edac" "entropy" "filefd" "filesystem" "hwmon" "loadavg" "meminfo" "netclass" "netdev" "netstat" "nvme" "os" "pressure" "rapl" "schedstat" "stat" "thermal_zone" "time" "uname" "vmstat" "zfs" # Disabled by default "cgroups" "drm" "ethtool" "logind" "wifi" ]; port = 9112; listenAddress = "127.0.0.1"; }; # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # enable = true; # port = 9119; # listenAddress = "127.0.0.1"; # }; # services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) { # enable = true; # port = 9121; # listenAddress = "127.0.0.1"; # statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers; # }; # dnsmasq = lib.mkIf config.services.dnsmasq.enable { # enable = true; # port = 9211; # listenAddress = "127.0.0.1"; # }; systemd = { enable = true; port = 9116; listenAddress = "127.0.0.1"; extraFlags = [ "--log.level=info" "--web.disable-exporter-metrics" # "--systemd.collector.enable-resolved" "--systemd.collector.enable-restart-count" # "--systemd.collector.enable-file-descriptor-size" "--systemd.collector.enable-ip-accounting" "--systemd.collector.unit-include=\".+\"" "--systemd.collector.unit-exclude=\".+\\\\.(device)\"" ]; }; }; }; }; }; }