aboutsummaryrefslogtreecommitdiffstats
path: root/modules/by-name/mo/monitoring/components
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--modules/by-name/mo/monitoring/components/grafana.nix51
-rw-r--r--modules/by-name/mo/monitoring/components/loki.nix130
-rw-r--r--modules/by-name/mo/monitoring/components/prometheus.nix120
-rw-r--r--modules/by-name/mo/monitoring/components/scrutiny.nix56
4 files changed, 246 insertions, 111 deletions
diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix
index 5459878..6fe5b70 100644
--- a/modules/by-name/mo/monitoring/components/grafana.nix
+++ b/modules/by-name/mo/monitoring/components/grafana.nix
@@ -118,31 +118,44 @@ in {
};
};
+ users = {
+ groups = {
+ grafana = {
+ gid = config.vhack.constants.ids.gids.grafana;
+ };
+ };
+ users = {
+ grafana = {
+ isSystemUser = true;
+ group = "grafana";
+ uid = lib.mkForce config.vhack.constants.ids.uids.grafana;
+ };
+ };
+ };
+
vhack = {
postgresql.enable = true;
+
+ persist.directories = [
+ {
+ directory = "/var/lib/grafana/";
+ user = "grafana";
+ group = "grafana";
+ mode = "0700";
+ }
+ ];
+
monitoring.grafana.dashboards = [
- ../dashboards/Errors.json
- ../dashboards/Performance.json
- ../dashboards/Scraping_Jobs.json
+ ../dashboards/node_exporter.json
+ ../dashboards/performance.json
+ ../dashboards/scraping_jobs.json
+ ../dashboards/systemd_exporter.json
];
+
+ anubis.instances."${cfg.fqdn}".target = "http://${config.services.grafana.settings.server.http_addr}:${toString cfg.port}";
};
services = {
- nginx = {
- virtualHosts.${cfg.fqdn} = {
- locations."/" = {
- proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
- proxyWebsockets = true;
- extraConfig = ''
- proxy_set_header Host $host;
- '';
- };
-
- forceSSL = true;
- enableACME = true;
- };
- };
-
postgresql = {
ensureUsers = [
{
@@ -198,7 +211,7 @@ in {
folder = "Vhack";
options.path = pkgs.symlinkJoin {
name = "dashboards";
- paths = map (p: pkgs.runCommand "davhackoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
+ paths = map (p: pkgs.runCommand "dashoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
};
allowUiUpdates = true;
disableDeletion = true;
diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix
index 9a285c5..149e577 100644
--- a/modules/by-name/mo/monitoring/components/loki.nix
+++ b/modules/by-name/mo/monitoring/components/loki.nix
@@ -17,50 +17,56 @@ in {
};
config = lib.mkIf cfg.enable {
- services = {
- loki = {
- enable = true;
- dataDir = "/var/lib/loki";
- package = pkgs.grafana-loki;
- configuration = {
- auth_enabled = false;
-
- server.http_listen_port = cfg.port;
-
- common = {
- ring = {
- instance_addr = "127.0.0.1";
- kvstore = {
- store = "inmemory";
- };
- };
-
- replication_factor = 1;
- path_prefix = "/tmp/loki";
- };
-
- schema_config = {
- configs = [
- {
- from = "2020-05-15";
- store = "tsdb";
- object_store = "filesystem";
- schema = "v13";
- index = {
- prefix = "index_";
- period = "24h";
- };
- }
- ];
- };
+ vhack = {
+ persist.directories = [
+ {
+ directory = "/var/lib/loki";
+ user = "loki";
+ group = "loki";
+ mode = "0700";
+ }
+ {
+ directory = "/var/lib/fluent-bit";
+ user = "fluent-bit";
+ group = "fluent-bit";
+ mode = "0700";
+ }
+ ];
+ };
- storage_config = {
- filesystem.directory = "/tmp/loki/chunks";
- };
+ users = {
+ groups = {
+ fluent-bit = {
+ gid = config.vhack.constants.ids.gids.fluent-bit;
+ };
+ loki = {
+ gid = config.vhack.constants.ids.gids.loki;
};
};
+ users = {
+ fluent-bit = {
+ isSystemUser = true;
+ group = "fluent-bit";
+ uid = config.vhack.constants.ids.uids.fluent-bit;
+ };
+ loki = {
+ isSystemUser = true;
+ group = "loki";
+ uid = config.vhack.constants.ids.uids.loki;
+ };
+ };
+ };
- # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+ # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+ systemd.services.fluent-bit = {
+ serviceConfig = {
+ StateDirectory = "fluent-bit";
+ User = "fluent-bit";
+ Group = "fluent-bit";
+ DynamicUser = lib.mkForce false;
+ };
+ };
+ services = {
fluent-bit = {
enable = true;
settings = {
@@ -85,7 +91,7 @@ in {
path = "/var/log/journal";
# Database file to keep track of the journald cursor.
- db = "/var/fluent-bit/systemd.db";
+ db = "/var/lib/fluent-bit/systemd.db";
# Start reading new entries. Skip entries already stored in journald.
read_from_tail = true;
@@ -117,6 +123,48 @@ in {
};
graceLimit = "1m";
};
+
+ loki = {
+ enable = true;
+ dataDir = "/var/lib/loki";
+ package = pkgs.grafana-loki;
+ configuration = {
+ auth_enabled = false;
+
+ server.http_listen_port = cfg.port;
+
+ common = {
+ ring = {
+ instance_addr = "127.0.0.1";
+ kvstore = {
+ store = "inmemory";
+ };
+ };
+
+ replication_factor = 1;
+ path_prefix = "/tmp/loki";
+ };
+
+ schema_config = {
+ configs = [
+ {
+ from = "2020-05-15";
+ store = "tsdb";
+ object_store = "filesystem";
+ schema = "v13";
+ index = {
+ prefix = "index_";
+ period = "24h";
+ };
+ }
+ ];
+ };
+
+ storage_config = {
+ filesystem.directory = "/tmp/loki/chunks";
+ };
+ };
+ };
};
};
}
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
index 468ece3..ca947d3 100644
--- a/modules/by-name/mo/monitoring/components/prometheus.nix
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -8,7 +8,6 @@
commonLabels = {
hostname = config.networking.hostName;
- inherit (config.vhack.monitoring.grafana) fqdn;
};
prom_cfg = config.services.prometheus;
@@ -22,6 +21,35 @@ in {
default = 3001;
};
+ remoteWriteReceiver = lib.mkOption {
+ example = "https://prometheus.server2.vhack.eu";
+ type = lib.types.nullOr lib.types.str;
+ default = null;
+ description = ''
+ If not null, this prometheus server will be exposed via a reverse proxy and allow
+ other prometheus servers to write metrics to it.
+ '';
+ };
+ remoteWriteTo = lib.mkOption {
+ example = "https://prometheus.server3.vhack.eu";
+ type = lib.types.nullOr (lib.types.submodule {
+ options = {
+ port = lib.mkOption {
+ type = lib.types.nullOr lib.types.port;
+ default = null;
+ };
+ url = lib.mkOption {
+ type = lib.types.str;
+ };
+ };
+ });
+ default = null;
+ description = ''
+ If not null, this prometheus server will run in agent mode, writing all recived
+ metrics to the specified remote prometheus server.
+ '';
+ };
+
sources = lib.mkOption {
type = lib.types.listOf (lib.types.submodule {
options = {
@@ -30,6 +58,7 @@ in {
};
target = lib.mkOption {
type = lib.types.str;
+ description = "The target url to scrape from.";
};
extra = lib.mkOption {
type = lib.types.attrsOf lib.types.anything;
@@ -42,16 +71,22 @@ in {
config = lib.mkIf cfg.enable {
vhack = {
nginx.enable = true;
+
+ persist.directories = [
+ {
+ directory = "/var/lib/prometheus2";
+ user = "prometheus";
+ group = "prometheus";
+ mode = "0700";
+ }
+ ];
+
monitoring.prometheus.sources = [
{
name = "node";
target = "127.0.0.1:${toString prom_cfg.exporters.node.port}";
}
{
- name = "smartctl";
- target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}";
- }
- {
name = "prometheus_internal";
target = "127.0.0.1:${toString prom_cfg.port}";
}
@@ -59,18 +94,36 @@ in {
name = "systemd";
target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}";
}
- {
- name = "netdata";
- target = "127.0.0.1:19999";
- extra = {
- metrics_path = "/api/v1/allmetrics";
- params.format = ["prometheus"];
- honor_labels = true;
- };
- }
];
};
+ services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) {
+ virtualHosts.${cfg.remoteWriteReceiver} = {
+ locations."/" = {
+ proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write";
+
+ recommendedProxySettings = true;
+ proxyWebsockets = true;
+ };
+
+ enableACME = true;
+ forceSSL = true;
+ };
+ };
+
+ users = {
+ groups = {
+ node-exporter.gid = config.vhack.constants.ids.gids.node-exporter;
+ };
+ users = {
+ node-exporter = {
+ isSystemUser = true;
+ group = "node-exporter";
+ uid = config.vhack.constants.ids.uids.node-exporter;
+ };
+ };
+ };
+
services = {
prometheus = {
enable = true;
@@ -79,6 +132,28 @@ in {
globalConfig = {
scrape_interval = "15s";
};
+ remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [
+ {
+ url = let
+ port =
+ if (cfg.remoteWriteTo.port != null)
+ then ":${toString cfg.remoteWriteTo.port}"
+ else "";
+ in "${cfg.remoteWriteTo.url}${port}";
+ # TODO: Needs RFC42 for prometheus <2026-07-19>
+ # protobuf_message = "io.prometheus.write.v2.Request";
+ }
+ ];
+
+ enableAgentMode = cfg.remoteWriteTo != null;
+ extraFlags = lib.mkMerge [
+ (
+ lib.mkIf (cfg.remoteWriteReceiver != null) [
+ "--web.enable-remote-write-receiver"
+ # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request"
+ ]
+ )
+ ];
scrapeConfigs = builtins.map (attr:
nixLib.warnMerge {
@@ -90,7 +165,7 @@ in {
}
];
}
- attr.extra "prometheus scrapeConfig")
+ attr.extra "prometheus scrapeConfig '${attr.name}'")
cfg.sources;
# ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
# job_name = "redis";
@@ -159,11 +234,6 @@ in {
port = 9112;
listenAddress = "127.0.0.1";
};
- smartctl = {
- enable = true;
- port = 9115;
- listenAddress = "127.0.0.1";
- };
# services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
# enable = true;
# port = 9119;
@@ -184,6 +254,16 @@ in {
enable = true;
port = 9116;
listenAddress = "127.0.0.1";
+ extraFlags = [
+ "--log.level=info"
+ "--web.disable-exporter-metrics"
+ # "--systemd.collector.enable-resolved"
+ "--systemd.collector.enable-restart-count"
+ # "--systemd.collector.enable-file-descriptor-size"
+ "--systemd.collector.enable-ip-accounting"
+ "--systemd.collector.unit-include=\".+\""
+ "--systemd.collector.unit-exclude=\".+\\\\.(device)\""
+ ];
};
};
};
diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix
index d69dcab..adb515e 100644
--- a/modules/by-name/mo/monitoring/components/scrutiny.nix
+++ b/modules/by-name/mo/monitoring/components/scrutiny.nix
@@ -8,18 +8,11 @@
in {
options.vhack.monitoring.scrutiny = {
enable = lib.mkEnableOption "scrutiny service";
-
- fqdn = lib.mkOption {
- type = lib.types.nullOr lib.types.str;
- description = ''
- If a string, this will be the domain under which the scrutiny web interface will be servced.
-
- If null, the web interface will not be served and only the prometheus metrics will be accessible.
- '';
- };
};
- config = lib.mkIf cfg.enable {
+ # We don't currently _care_ about the health of the hardware, because hetzner does
+ # this for us (nor can we care, because it's not exposed in the VPS)
+ config = lib.mkIf false {
vhack.monitoring = {
prometheus.sources = [
{
@@ -27,12 +20,29 @@ in {
extra.metrics_path = "/api/metrics";
target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
}
+ {
+ name = "smartctl";
+ target = "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}";
+ }
];
grafana.dashboards = [
../dashboards/Health.json
];
};
+ users = {
+ groups = {
+ smartctl-exporter-access.gid = config.vhack.constants.ids.gids.smartctl-exporter-access;
+ };
+ users = {
+ smartctl-exporter-access = {
+ isSystemUser = true;
+ group = "smartctl-exporter-access";
+ uid = config.vhack.constants.ids.uids.smartctl-exporter-access;
+ };
+ };
+ };
+
services = {
scrutiny = {
enable = true;
@@ -62,27 +72,11 @@ in {
enable = true;
};
};
-
- # nginx.statusPage = lib.mkDefault config.services.nginx.enable;
- nginx = lib.mkIf (cfg.fqdn != null) {
- virtualHosts."${cfg.fqdn}" = {
- locations."/".proxyPass = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
-
- enableACME = true;
- forceSSL = true;
- };
- };
-
- netdata = {
- enable = true;
- config = {
- # web.mode = "none";
- # web."bind to" = "127.0.0.1:19999";
- global = {
- "debug log" = "syslog";
- "access log" = "syslog";
- "error log" = "syslog";
- };
+ prometheus = {
+ exporters.smartctl = {
+ enable = true;
+ port = 9115;
+ listenAddress = "127.0.0.1";
};
};
};
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-21 06:22:22 +0000