diff options
Diffstat (limited to 'modules/by-name/mo/monitoring/components')
| -rw-r--r-- | modules/by-name/mo/monitoring/components/grafana.nix | 51 | ||||
| -rw-r--r-- | modules/by-name/mo/monitoring/components/loki.nix | 130 | ||||
| -rw-r--r-- | modules/by-name/mo/monitoring/components/prometheus.nix | 120 | ||||
| -rw-r--r-- | modules/by-name/mo/monitoring/components/scrutiny.nix | 56 |
4 files changed, 246 insertions, 111 deletions
diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix index 5459878..6fe5b70 100644 --- a/modules/by-name/mo/monitoring/components/grafana.nix +++ b/modules/by-name/mo/monitoring/components/grafana.nix @@ -118,31 +118,44 @@ in { }; }; + users = { + groups = { + grafana = { + gid = config.vhack.constants.ids.gids.grafana; + }; + }; + users = { + grafana = { + isSystemUser = true; + group = "grafana"; + uid = lib.mkForce config.vhack.constants.ids.uids.grafana; + }; + }; + }; + vhack = { postgresql.enable = true; + + persist.directories = [ + { + directory = "/var/lib/grafana/"; + user = "grafana"; + group = "grafana"; + mode = "0700"; + } + ]; + monitoring.grafana.dashboards = [ - ../dashboards/Errors.json - ../dashboards/Performance.json - ../dashboards/Scraping_Jobs.json + ../dashboards/node_exporter.json + ../dashboards/performance.json + ../dashboards/scraping_jobs.json + ../dashboards/systemd_exporter.json ]; + + anubis.instances."${cfg.fqdn}".target = "http://${config.services.grafana.settings.server.http_addr}:${toString cfg.port}"; }; services = { - nginx = { - virtualHosts.${cfg.fqdn} = { - locations."/" = { - proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}"; - proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - ''; - }; - - forceSSL = true; - enableACME = true; - }; - }; - postgresql = { ensureUsers = [ { @@ -198,7 +211,7 @@ in { folder = "Vhack"; options.path = pkgs.symlinkJoin { name = "dashboards"; - paths = map (p: pkgs.runCommand "davhackoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards; + paths = map (p: pkgs.runCommand "dashoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards; }; allowUiUpdates = true; disableDeletion = true; diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix index 9a285c5..149e577 100644 --- a/modules/by-name/mo/monitoring/components/loki.nix +++ b/modules/by-name/mo/monitoring/components/loki.nix @@ -17,50 +17,56 @@ in { }; config = lib.mkIf cfg.enable { - services = { - loki = { - enable = true; - dataDir = "/var/lib/loki"; - package = pkgs.grafana-loki; - configuration = { - auth_enabled = false; - - server.http_listen_port = cfg.port; - - common = { - ring = { - instance_addr = "127.0.0.1"; - kvstore = { - store = "inmemory"; - }; - }; - - replication_factor = 1; - path_prefix = "/tmp/loki"; - }; - - schema_config = { - configs = [ - { - from = "2020-05-15"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - index = { - prefix = "index_"; - period = "24h"; - }; - } - ]; - }; + vhack = { + persist.directories = [ + { + directory = "/var/lib/loki"; + user = "loki"; + group = "loki"; + mode = "0700"; + } + { + directory = "/var/lib/fluent-bit"; + user = "fluent-bit"; + group = "fluent-bit"; + mode = "0700"; + } + ]; + }; - storage_config = { - filesystem.directory = "/tmp/loki/chunks"; - }; + users = { + groups = { + fluent-bit = { + gid = config.vhack.constants.ids.gids.fluent-bit; + }; + loki = { + gid = config.vhack.constants.ids.gids.loki; }; }; + users = { + fluent-bit = { + isSystemUser = true; + group = "fluent-bit"; + uid = config.vhack.constants.ids.uids.fluent-bit; + }; + loki = { + isSystemUser = true; + group = "loki"; + uid = config.vhack.constants.ids.uids.loki; + }; + }; + }; - # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline + # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline + systemd.services.fluent-bit = { + serviceConfig = { + StateDirectory = "fluent-bit"; + User = "fluent-bit"; + Group = "fluent-bit"; + DynamicUser = lib.mkForce false; + }; + }; + services = { fluent-bit = { enable = true; settings = { @@ -85,7 +91,7 @@ in { path = "/var/log/journal"; # Database file to keep track of the journald cursor. - db = "/var/fluent-bit/systemd.db"; + db = "/var/lib/fluent-bit/systemd.db"; # Start reading new entries. Skip entries already stored in journald. read_from_tail = true; @@ -117,6 +123,48 @@ in { }; graceLimit = "1m"; }; + + loki = { + enable = true; + dataDir = "/var/lib/loki"; + package = pkgs.grafana-loki; + configuration = { + auth_enabled = false; + + server.http_listen_port = cfg.port; + + common = { + ring = { + instance_addr = "127.0.0.1"; + kvstore = { + store = "inmemory"; + }; + }; + + replication_factor = 1; + path_prefix = "/tmp/loki"; + }; + + schema_config = { + configs = [ + { + from = "2020-05-15"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + filesystem.directory = "/tmp/loki/chunks"; + }; + }; + }; }; }; } diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix index 468ece3..ca947d3 100644 --- a/modules/by-name/mo/monitoring/components/prometheus.nix +++ b/modules/by-name/mo/monitoring/components/prometheus.nix @@ -8,7 +8,6 @@ commonLabels = { hostname = config.networking.hostName; - inherit (config.vhack.monitoring.grafana) fqdn; }; prom_cfg = config.services.prometheus; @@ -22,6 +21,35 @@ in { default = 3001; }; + remoteWriteReceiver = lib.mkOption { + example = "https://prometheus.server2.vhack.eu"; + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + If not null, this prometheus server will be exposed via a reverse proxy and allow + other prometheus servers to write metrics to it. + ''; + }; + remoteWriteTo = lib.mkOption { + example = "https://prometheus.server3.vhack.eu"; + type = lib.types.nullOr (lib.types.submodule { + options = { + port = lib.mkOption { + type = lib.types.nullOr lib.types.port; + default = null; + }; + url = lib.mkOption { + type = lib.types.str; + }; + }; + }); + default = null; + description = '' + If not null, this prometheus server will run in agent mode, writing all recived + metrics to the specified remote prometheus server. + ''; + }; + sources = lib.mkOption { type = lib.types.listOf (lib.types.submodule { options = { @@ -30,6 +58,7 @@ in { }; target = lib.mkOption { type = lib.types.str; + description = "The target url to scrape from."; }; extra = lib.mkOption { type = lib.types.attrsOf lib.types.anything; @@ -42,16 +71,22 @@ in { config = lib.mkIf cfg.enable { vhack = { nginx.enable = true; + + persist.directories = [ + { + directory = "/var/lib/prometheus2"; + user = "prometheus"; + group = "prometheus"; + mode = "0700"; + } + ]; + monitoring.prometheus.sources = [ { name = "node"; target = "127.0.0.1:${toString prom_cfg.exporters.node.port}"; } { - name = "smartctl"; - target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}"; - } - { name = "prometheus_internal"; target = "127.0.0.1:${toString prom_cfg.port}"; } @@ -59,18 +94,36 @@ in { name = "systemd"; target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}"; } - { - name = "netdata"; - target = "127.0.0.1:19999"; - extra = { - metrics_path = "/api/v1/allmetrics"; - params.format = ["prometheus"]; - honor_labels = true; - }; - } ]; }; + services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) { + virtualHosts.${cfg.remoteWriteReceiver} = { + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write"; + + recommendedProxySettings = true; + proxyWebsockets = true; + }; + + enableACME = true; + forceSSL = true; + }; + }; + + users = { + groups = { + node-exporter.gid = config.vhack.constants.ids.gids.node-exporter; + }; + users = { + node-exporter = { + isSystemUser = true; + group = "node-exporter"; + uid = config.vhack.constants.ids.uids.node-exporter; + }; + }; + }; + services = { prometheus = { enable = true; @@ -79,6 +132,28 @@ in { globalConfig = { scrape_interval = "15s"; }; + remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [ + { + url = let + port = + if (cfg.remoteWriteTo.port != null) + then ":${toString cfg.remoteWriteTo.port}" + else ""; + in "${cfg.remoteWriteTo.url}${port}"; + # TODO: Needs RFC42 for prometheus <2026-07-19> + # protobuf_message = "io.prometheus.write.v2.Request"; + } + ]; + + enableAgentMode = cfg.remoteWriteTo != null; + extraFlags = lib.mkMerge [ + ( + lib.mkIf (cfg.remoteWriteReceiver != null) [ + "--web.enable-remote-write-receiver" + # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request" + ] + ) + ]; scrapeConfigs = builtins.map (attr: nixLib.warnMerge { @@ -90,7 +165,7 @@ in { } ]; } - attr.extra "prometheus scrapeConfig") + attr.extra "prometheus scrapeConfig '${attr.name}'") cfg.sources; # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # job_name = "redis"; @@ -159,11 +234,6 @@ in { port = 9112; listenAddress = "127.0.0.1"; }; - smartctl = { - enable = true; - port = 9115; - listenAddress = "127.0.0.1"; - }; # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # enable = true; # port = 9119; @@ -184,6 +254,16 @@ in { enable = true; port = 9116; listenAddress = "127.0.0.1"; + extraFlags = [ + "--log.level=info" + "--web.disable-exporter-metrics" + # "--systemd.collector.enable-resolved" + "--systemd.collector.enable-restart-count" + # "--systemd.collector.enable-file-descriptor-size" + "--systemd.collector.enable-ip-accounting" + "--systemd.collector.unit-include=\".+\"" + "--systemd.collector.unit-exclude=\".+\\\\.(device)\"" + ]; }; }; }; diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix index d69dcab..adb515e 100644 --- a/modules/by-name/mo/monitoring/components/scrutiny.nix +++ b/modules/by-name/mo/monitoring/components/scrutiny.nix @@ -8,18 +8,11 @@ in { options.vhack.monitoring.scrutiny = { enable = lib.mkEnableOption "scrutiny service"; - - fqdn = lib.mkOption { - type = lib.types.nullOr lib.types.str; - description = '' - If a string, this will be the domain under which the scrutiny web interface will be servced. - - If null, the web interface will not be served and only the prometheus metrics will be accessible. - ''; - }; }; - config = lib.mkIf cfg.enable { + # We don't currently _care_ about the health of the hardware, because hetzner does + # this for us (nor can we care, because it's not exposed in the VPS) + config = lib.mkIf false { vhack.monitoring = { prometheus.sources = [ { @@ -27,12 +20,29 @@ in { extra.metrics_path = "/api/metrics"; target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}"; } + { + name = "smartctl"; + target = "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"; + } ]; grafana.dashboards = [ ../dashboards/Health.json ]; }; + users = { + groups = { + smartctl-exporter-access.gid = config.vhack.constants.ids.gids.smartctl-exporter-access; + }; + users = { + smartctl-exporter-access = { + isSystemUser = true; + group = "smartctl-exporter-access"; + uid = config.vhack.constants.ids.uids.smartctl-exporter-access; + }; + }; + }; + services = { scrutiny = { enable = true; @@ -62,27 +72,11 @@ in { enable = true; }; }; - - # nginx.statusPage = lib.mkDefault config.services.nginx.enable; - nginx = lib.mkIf (cfg.fqdn != null) { - virtualHosts."${cfg.fqdn}" = { - locations."/".proxyPass = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}"; - - enableACME = true; - forceSSL = true; - }; - }; - - netdata = { - enable = true; - config = { - # web.mode = "none"; - # web."bind to" = "127.0.0.1:19999"; - global = { - "debug log" = "syslog"; - "access log" = "syslog"; - "error log" = "syslog"; - }; + prometheus = { + exporters.smartctl = { + enable = true; + port = 9115; + listenAddress = "127.0.0.1"; }; }; }; |