aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--hosts/by-name/server2/configuration.nix12
-rw-r--r--hosts/by-name/server2/secrets/grafana/adminPassword.age17
-rw-r--r--hosts/by-name/server2/secrets/grafana/secretKey.age18
-rw-r--r--modules/by-name/co/constants/module.nix10
-rw-r--r--modules/by-name/mo/monitoring/components/grafana.nix44
-rw-r--r--modules/by-name/mo/monitoring/components/loki.nix130
-rw-r--r--modules/by-name/mo/monitoring/components/prometheus.nix48
-rw-r--r--modules/by-name/mo/monitoring/components/scrutiny.nix56
-rw-r--r--modules/by-name/ng/nginx/module.nix4
-rw-r--r--zones/vhack.eu/zone.nix4
10 files changed, 236 insertions, 107 deletions
diff --git a/hosts/by-name/server2/configuration.nix b/hosts/by-name/server2/configuration.nix
index fd00c02..2e04324 100644
--- a/hosts/by-name/server2/configuration.nix
+++ b/hosts/by-name/server2/configuration.nix
@@ -26,6 +26,18 @@
privatePassword = ./secrets/backup/backuppass.age;
user = "u384702-sub3";
};
+
+ monitoring = {
+ enable = true;
+ grafana = {
+ fqdn = "grafana.vhack.eu";
+
+ contactPoints = ["admin@vhack.eu"];
+ adminPassword = ./secrets/grafana/adminPassword.age;
+ secretKey = ./secrets/grafana/secretKey.age;
+ };
+ };
+
dns = {
enable = true;
openFirewall = true;
diff --git a/hosts/by-name/server2/secrets/grafana/adminPassword.age b/hosts/by-name/server2/secrets/grafana/adminPassword.age
new file mode 100644
index 0000000..ff072fc
--- /dev/null
+++ b/hosts/by-name/server2/secrets/grafana/adminPassword.age
@@ -0,0 +1,17 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5bi9zRm5RRmU0YjJ4N3pi
+S2lNNGZRNUpycnUrZ3VRdFBsdkdIRjVTakRFCkxtVG1VbUFTbHpBZlYxV2pMSU5p
+a0lRcGVPWUUwMWpKdm5oYktWRklyWEUKLT4gWDI1NTE5IHNzN2NXRWYxdjladFF6
+ejk4eW43eUFsSStIV3FpWVJZREhLVEpjRkkvWFkKdlZtOG5pbDVvd0hwUEdMZnVn
+WFAvYWJDUEVBckc3M3NVQUdvdFVqNDB0QQotPiBzc2gtZWQyNTUxOSBYUG94RFEg
+RkpweWRJQklNUHVRWUhicXdDUVc3cXhyRi9KOS9wYWxDYnNNcjIzWmhERQpZZnpr
+eHRWbnI0cFd1T3h3TTVGZ1JiUzUrNUpSNkUwM2UyWTVYZ2VHVHBnCi0+IC5eKT0+
+eS1ncmVhc2Ugc0YvI25zZyAub3YgPDNtbmsKejExd2NBV0E2Q1pLSVhyQTRLaWZ6
+dmZSZHhidjBqeW10RnIxNStVTGIyY1RMS25xaUlxRDVxMlpKQQotLS0gc3BER0to
+dGdicEd3d1QzWWdoTmpkM1V5L0VvVG82cnNGY0hJTWUwVVBmTQpRQwKmFmQj10Z1
+AXllRSWcTSCzcM7fCVjCRcTF2lsxtXiXUkTS59yfNih02myuYCFIUG/MDl3A3pur
+WckFpQVHTdvQ9ilYORI/zi9LTmPy8TZYYcdmXnNRelfZ3fAqiR8ScIEG3P0smNyP
+9h6BcfiWYG2ppANLOeZxHz/UoG/LORXJyXoTCuiG8KJ714jX8f3CMX3+m9q/GmAX
+VZiYvvbGgrN5T5QrZE+8quRM1laAXmnmbDynODfhNFNw4/I0Opi0CF5MSObmvzKN
+z6vsc/kTGMqbip6k/LKvzea+jXMc8b8BeO0Y
+-----END AGE ENCRYPTED FILE-----
diff --git a/hosts/by-name/server2/secrets/grafana/secretKey.age b/hosts/by-name/server2/secrets/grafana/secretKey.age
new file mode 100644
index 0000000..bfe4afe
--- /dev/null
+++ b/hosts/by-name/server2/secrets/grafana/secretKey.age
@@ -0,0 +1,18 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsYlhkcFdwL2NDUllGVko3
+bTEwVE00MjRoZUpXMGFWZFFSMXNBUkpFRVU4CkJwSU5RUnlITnRSZklVZjNBN3Ix
+NDdUS0ZpWVNBWEY3MVZYVlBkZkM3VlUKLT4gWDI1NTE5IHdsSm55NCtIT0Z4cUhx
+dE9pM0RmSHUrUGZTYXhuQlM4TGVKcm5MWGZNekkKQXZmZ05JVmZYdzJIYlN5WmxV
+Zlh0TWlxWWZucldweHpjdFM5TGJKZnprOAotPiBzc2gtZWQyNTUxOSBYUG94RFEg
+SjVTVXVJRjQrc0RIenVIVmNmZEtQS1lWbVNhSzY5QjJXRUxNY1JvVGdIOAo5V0Y5
+SGhTRlZUcFE3eU1iazBwYzh0Q2UxTnA2UTJQQ2VhaW1INnBGTTQwCi0+IGdeazN5
+ZmstZ3JlYXNlIGpZSGUgTUYja2N5RCAhICNaeT8KVnFRWjJXQ1lMekJQZG5jbGk4
+KzB5YVVmMGRFT1pUZzF3dkJBby94NXZjbkFOYlQ1dy9VWUFnaVRlTHpkUXdZaQox
+R1BMN3RjVXFwbitqU1pETzdlc0VqOFZhbVBrRlFiOWl3TWZkNzlCSS9kNkhBVUcw
+Zk5UbFRrZXhWQ2QrZwotLS0gbFV3MXRoYjZ4QVo0UU9iMmpGVllNcDdJZFQvUEFR
+UXpWYmptWmdWSllLVQrTggsrW+giXmbRL6Jq3VqT8uXhqHs5ZTCnPQSpKth04fzr
+mNLKMIi2kRJm3hJCaSPBrLpDuop+08hjGe0l0CxvuRlnFUOZkh+C/kFWOI3ecaWt
+TYVDkKNLXjrhyfw/rVHy03czluQncFYHFKib8djDqYTUa0ORj+M5cIgAzWODyCWB
+IGftRRjhDMHhQzq0GRwwu6jNA+ovT1cN1aMmafQFmqTR7BqO18HqKsGdtirFLV3o
+qrvHjqz1cJNKYEU89ogLJydTJHyuPhuPeHBl
+-----END AGE ENCRYPTED FILE-----
diff --git a/modules/by-name/co/constants/module.nix b/modules/by-name/co/constants/module.nix
index b94020b..9127488 100644
--- a/modules/by-name/co/constants/module.nix
+++ b/modules/by-name/co/constants/module.nix
@@ -56,6 +56,11 @@
anubis = 342;
postfix-tlspol = 343;
rocie = 344;
+ fluent-bit = 345;
+ loki = 346;
+ grafana = 347;
+ node-exporter = 348;
+ smartctl-exporter-access = 349;
# As per the NixOS file, the uids should not be greater or equal to 400;
};
@@ -96,6 +101,11 @@
resolvconf # This group is not matched to an user?
stalwart-mail-certificates # This group is used to connect nginx and stalwart-mail
rocie
+ fluent-bit
+ loki
+ grafana
+ node-exporter
+ smartctl-exporter-access
;
# The gid should match the uid. Thus should not be >= 400;
diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix
index 5459878..534ac78 100644
--- a/modules/by-name/mo/monitoring/components/grafana.nix
+++ b/modules/by-name/mo/monitoring/components/grafana.nix
@@ -118,31 +118,43 @@ in {
};
};
+ users = {
+ groups = {
+ grafana = {
+ gid = config.vhack.constants.ids.gids.grafana;
+ };
+ };
+ users = {
+ grafana = {
+ isSystemUser = true;
+ group = "grafana";
+ uid = lib.mkForce config.vhack.constants.ids.uids.grafana;
+ };
+ };
+ };
+
vhack = {
postgresql.enable = true;
+
+ persist.directories = [
+ {
+ directory = "/var/lib/grafana/";
+ user = "grafana";
+ group = "grafana";
+ mode = "0700";
+ }
+ ];
+
monitoring.grafana.dashboards = [
../dashboards/Errors.json
../dashboards/Performance.json
../dashboards/Scraping_Jobs.json
];
+
+ anubis.instances."${cfg.fqdn}".target = "http://${config.services.grafana.settings.server.http_addr}:${toString cfg.port}";
};
services = {
- nginx = {
- virtualHosts.${cfg.fqdn} = {
- locations."/" = {
- proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
- proxyWebsockets = true;
- extraConfig = ''
- proxy_set_header Host $host;
- '';
- };
-
- forceSSL = true;
- enableACME = true;
- };
- };
-
postgresql = {
ensureUsers = [
{
@@ -198,7 +210,7 @@ in {
folder = "Vhack";
options.path = pkgs.symlinkJoin {
name = "dashboards";
- paths = map (p: pkgs.runCommand "davhackoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
+ paths = map (p: pkgs.runCommand "dashoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
};
allowUiUpdates = true;
disableDeletion = true;
diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix
index 9a285c5..149e577 100644
--- a/modules/by-name/mo/monitoring/components/loki.nix
+++ b/modules/by-name/mo/monitoring/components/loki.nix
@@ -17,50 +17,56 @@ in {
};
config = lib.mkIf cfg.enable {
- services = {
- loki = {
- enable = true;
- dataDir = "/var/lib/loki";
- package = pkgs.grafana-loki;
- configuration = {
- auth_enabled = false;
-
- server.http_listen_port = cfg.port;
-
- common = {
- ring = {
- instance_addr = "127.0.0.1";
- kvstore = {
- store = "inmemory";
- };
- };
-
- replication_factor = 1;
- path_prefix = "/tmp/loki";
- };
-
- schema_config = {
- configs = [
- {
- from = "2020-05-15";
- store = "tsdb";
- object_store = "filesystem";
- schema = "v13";
- index = {
- prefix = "index_";
- period = "24h";
- };
- }
- ];
- };
+ vhack = {
+ persist.directories = [
+ {
+ directory = "/var/lib/loki";
+ user = "loki";
+ group = "loki";
+ mode = "0700";
+ }
+ {
+ directory = "/var/lib/fluent-bit";
+ user = "fluent-bit";
+ group = "fluent-bit";
+ mode = "0700";
+ }
+ ];
+ };
- storage_config = {
- filesystem.directory = "/tmp/loki/chunks";
- };
+ users = {
+ groups = {
+ fluent-bit = {
+ gid = config.vhack.constants.ids.gids.fluent-bit;
+ };
+ loki = {
+ gid = config.vhack.constants.ids.gids.loki;
};
};
+ users = {
+ fluent-bit = {
+ isSystemUser = true;
+ group = "fluent-bit";
+ uid = config.vhack.constants.ids.uids.fluent-bit;
+ };
+ loki = {
+ isSystemUser = true;
+ group = "loki";
+ uid = config.vhack.constants.ids.uids.loki;
+ };
+ };
+ };
- # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+ # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+ systemd.services.fluent-bit = {
+ serviceConfig = {
+ StateDirectory = "fluent-bit";
+ User = "fluent-bit";
+ Group = "fluent-bit";
+ DynamicUser = lib.mkForce false;
+ };
+ };
+ services = {
fluent-bit = {
enable = true;
settings = {
@@ -85,7 +91,7 @@ in {
path = "/var/log/journal";
# Database file to keep track of the journald cursor.
- db = "/var/fluent-bit/systemd.db";
+ db = "/var/lib/fluent-bit/systemd.db";
# Start reading new entries. Skip entries already stored in journald.
read_from_tail = true;
@@ -117,6 +123,48 @@ in {
};
graceLimit = "1m";
};
+
+ loki = {
+ enable = true;
+ dataDir = "/var/lib/loki";
+ package = pkgs.grafana-loki;
+ configuration = {
+ auth_enabled = false;
+
+ server.http_listen_port = cfg.port;
+
+ common = {
+ ring = {
+ instance_addr = "127.0.0.1";
+ kvstore = {
+ store = "inmemory";
+ };
+ };
+
+ replication_factor = 1;
+ path_prefix = "/tmp/loki";
+ };
+
+ schema_config = {
+ configs = [
+ {
+ from = "2020-05-15";
+ store = "tsdb";
+ object_store = "filesystem";
+ schema = "v13";
+ index = {
+ prefix = "index_";
+ period = "24h";
+ };
+ }
+ ];
+ };
+
+ storage_config = {
+ filesystem.directory = "/tmp/loki/chunks";
+ };
+ };
+ };
};
};
}
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
index 468ece3..dbe6c06 100644
--- a/modules/by-name/mo/monitoring/components/prometheus.nix
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -42,16 +42,22 @@ in {
config = lib.mkIf cfg.enable {
vhack = {
nginx.enable = true;
+
+ persist.directories = [
+ {
+ directory = "/var/lib/prometheus2";
+ user = "prometheus";
+ group = "prometheus";
+ mode = "0700";
+ }
+ ];
+
monitoring.prometheus.sources = [
{
name = "node";
target = "127.0.0.1:${toString prom_cfg.exporters.node.port}";
}
{
- name = "smartctl";
- target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}";
- }
- {
name = "prometheus_internal";
target = "127.0.0.1:${toString prom_cfg.port}";
}
@@ -59,18 +65,22 @@ in {
name = "systemd";
target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}";
}
- {
- name = "netdata";
- target = "127.0.0.1:19999";
- extra = {
- metrics_path = "/api/v1/allmetrics";
- params.format = ["prometheus"];
- honor_labels = true;
- };
- }
];
};
+ users = {
+ groups = {
+ node-exporter.gid = config.vhack.constants.ids.gids.node-exporter;
+ };
+ users = {
+ node-exporter = {
+ isSystemUser = true;
+ group = "node-exporter";
+ uid = config.vhack.constants.ids.uids.node-exporter;
+ };
+ };
+ };
+
services = {
prometheus = {
enable = true;
@@ -159,11 +169,6 @@ in {
port = 9112;
listenAddress = "127.0.0.1";
};
- smartctl = {
- enable = true;
- port = 9115;
- listenAddress = "127.0.0.1";
- };
# services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
# enable = true;
# port = 9119;
@@ -184,6 +189,13 @@ in {
enable = true;
port = 9116;
listenAddress = "127.0.0.1";
+ extraFlags = [
+ "--systemd.collector.enable-restart-count"
+ "--systemd.collector.enable-file-descriptor-size"
+ "--systemd.collector.enable-ip-accounting"
+ "--log.level=info"
+ "--systemd.collector.unit-include=.*.service|.*.timer"
+ ];
};
};
};
diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix
index 31871cd..adb515e 100644
--- a/modules/by-name/mo/monitoring/components/scrutiny.nix
+++ b/modules/by-name/mo/monitoring/components/scrutiny.nix
@@ -8,18 +8,11 @@
in {
options.vhack.monitoring.scrutiny = {
enable = lib.mkEnableOption "scrutiny service";
-
- fqdn = lib.mkOption {
- type = lib.types.nullOr lib.types.str;
- description = ''
- If a string, this will be the domain under which the scrutiny web interface will be servced.
-
- If null, the web interface will not be served and only the prometheus metrics will be accessible.
- '';
- };
};
- config = {
+ # We don't currently _care_ about the health of the hardware, because hetzner does
+ # this for us (nor can we care, because it's not exposed in the VPS)
+ config = lib.mkIf false {
vhack.monitoring = {
prometheus.sources = [
{
@@ -27,12 +20,29 @@ in {
extra.metrics_path = "/api/metrics";
target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
}
+ {
+ name = "smartctl";
+ target = "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}";
+ }
];
grafana.dashboards = [
../dashboards/Health.json
];
};
+ users = {
+ groups = {
+ smartctl-exporter-access.gid = config.vhack.constants.ids.gids.smartctl-exporter-access;
+ };
+ users = {
+ smartctl-exporter-access = {
+ isSystemUser = true;
+ group = "smartctl-exporter-access";
+ uid = config.vhack.constants.ids.uids.smartctl-exporter-access;
+ };
+ };
+ };
+
services = {
scrutiny = {
enable = true;
@@ -62,27 +72,11 @@ in {
enable = true;
};
};
-
- # nginx.statusPage = lib.mkDefault config.services.nginx.enable;
- nginx = lib.mkIf (cfg.fqdn != null) {
- virtualHosts."${cfg.fqdn}" = {
- locations."/".proxyPass = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
-
- enableACME = true;
- forceSSL = true;
- };
- };
-
- netdata = {
- enable = true;
- config = {
- # web.mode = "none";
- # web."bind to" = "127.0.0.1:19999";
- global = {
- "debug log" = "syslog";
- "access log" = "syslog";
- "error log" = "syslog";
- };
+ prometheus = {
+ exporters.smartctl = {
+ enable = true;
+ port = 9115;
+ listenAddress = "127.0.0.1";
};
};
};
diff --git a/modules/by-name/ng/nginx/module.nix b/modules/by-name/ng/nginx/module.nix
index 734a2d2..096c09a 100644
--- a/modules/by-name/ng/nginx/module.nix
+++ b/modules/by-name/ng/nginx/module.nix
@@ -86,6 +86,10 @@ in {
};
services.nginx = {
enable = true;
+
+ # Enable the status page for the prometheus exporter.
+ statusPage = lib.mkIf config.services.prometheus.enable true;
+
virtualHosts = redirects;
};
};
diff --git a/zones/vhack.eu/zone.nix b/zones/vhack.eu/zone.nix
index 0e018e9..c528c54 100644
--- a/zones/vhack.eu/zone.nix
+++ b/zones/vhack.eu/zone.nix
@@ -38,7 +38,7 @@ in {
adminEmail = "dns-admin@foss-syndicate.org";
# NOTE(@bpeetz): ALWAYS change the serial number, when you change something in the
# zone file! <2025-04-01>
- serial = mkSerial 2025 08 03 01;
+ serial = mkSerial 2026 07 19 01;
};
useOrigin = false;
@@ -149,6 +149,8 @@ in {
taskchampion.CNAME = ["server2.vhack.eu."];
atuin-sync.CNAME = ["server2.vhack.eu."];
+ grafana.CNAME = ["server2.vhack.eu."];
+
dav.CNAME = ["server2.vhack.eu."];
etebase.CNAME = ["server2.vhack.eu."];
git.CNAME = ["server2.vhack.eu."];
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-19 17:18:49 +0000