aboutsummaryrefslogtreecommitdiffstats
path: root/modules/by-name/mo/monitoring/components
diff options
context:
space:
mode:
Diffstat (limited to 'modules/by-name/mo/monitoring/components')
-rw-r--r--modules/by-name/mo/monitoring/components/grafana.nix309
-rw-r--r--modules/by-name/mo/monitoring/components/loki.nix170
-rw-r--r--modules/by-name/mo/monitoring/components/perses.nix17
-rw-r--r--modules/by-name/mo/monitoring/components/prometheus.nix272
-rw-r--r--modules/by-name/mo/monitoring/components/scrutiny.nix100
5 files changed, 0 insertions, 868 deletions
diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix
deleted file mode 100644
index 6fe5b70..0000000
--- a/modules/by-name/mo/monitoring/components/grafana.nix
+++ /dev/null
@@ -1,309 +0,0 @@
-{
- config,
- lib,
- pkgs,
- ...
-}: let
- cfg = config.vhack.monitoring.grafana;
-in {
- options.vhack.monitoring.grafana = {
- enable = lib.mkEnableOption "grafana";
-
- orgId = lib.mkOption {
- type = lib.types.int;
- description = "Org ID where all vhack related config will be stored.";
- default = 1;
- };
-
- contactPoints = lib.mkOption {
- type = lib.types.listOf lib.types.str;
- description = "List of email addresses to send alerts to";
- default = [];
- };
-
- dashboards = lib.mkOption {
- type = lib.types.listOf lib.types.path;
- description = "dashboards to provision under 'Vhack' folder.";
- default = [];
- };
-
- fqdn = lib.mkOption {
- type = lib.types.str;
- description = "Domain under which Grafana will be served.";
- example = "grafana.example.org";
- };
-
- adminPassword = lib.mkOption {
- type = lib.types.path;
- description = "The age-encrypted initial admin password; passed to agenix";
- };
-
- secretKey = lib.mkOption {
- type = lib.types.path;
- description = "The age-encrypted secret key, used for signing; passed to agenix";
- };
-
- port = lib.mkOption {
- type = lib.types.port;
- description = "Port where Grafana listens to HTTP requests.";
- default = 3000;
- };
-
- debugLog = lib.mkOption {
- type = lib.types.bool;
- description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
- default = false;
- example = true;
- };
-
- smtp = lib.mkOption {
- description = "SMTP options.";
- default = null;
- type = lib.types.nullOr (
- lib.types.submodule {
- options = {
- from_address = lib.mkOption {
- type = lib.types.str;
- description = "SMTP address from which the emails originate.";
- example = "vaultwarden@mydomain.com";
- };
- from_name = lib.mkOption {
- type = lib.types.str;
- description = "SMTP name from which the emails originate.";
- default = "Grafana";
- };
- host = lib.mkOption {
- type = lib.types.str;
- description = "SMTP host to send the emails to.";
- };
- port = lib.mkOption {
- type = lib.types.port;
- description = "SMTP port to send the emails to.";
- default = 25;
- };
- username = lib.mkOption {
- type = lib.types.str;
- description = "Username to connect to the SMTP host.";
- };
- passwordFile = lib.mkOption {
- type = lib.types.str;
- description = "File containing the password to connect to the SMTP host.";
- };
- };
- }
- );
- };
- };
-
- config = lib.mkIf cfg.enable {
- assertions = [
- {
- assertion = builtins.length cfg.contactPoints > 0;
- message = "Must have at least one contact point for alerting";
- }
- ];
-
- age.secrets = {
- grafanaInitialAdminPw = {
- file = cfg.adminPassword;
- mode = "0700";
- owner = "grafana";
- group = "grafana";
- };
- grafanaSecretKey = {
- file = cfg.secretKey;
- mode = "0700";
- owner = "grafana";
- group = "grafana";
- };
- };
-
- users = {
- groups = {
- grafana = {
- gid = config.vhack.constants.ids.gids.grafana;
- };
- };
- users = {
- grafana = {
- isSystemUser = true;
- group = "grafana";
- uid = lib.mkForce config.vhack.constants.ids.uids.grafana;
- };
- };
- };
-
- vhack = {
- postgresql.enable = true;
-
- persist.directories = [
- {
- directory = "/var/lib/grafana/";
- user = "grafana";
- group = "grafana";
- mode = "0700";
- }
- ];
-
- monitoring.grafana.dashboards = [
- ../dashboards/node_exporter.json
- ../dashboards/performance.json
- ../dashboards/scraping_jobs.json
- ../dashboards/systemd_exporter.json
- ];
-
- anubis.instances."${cfg.fqdn}".target = "http://${config.services.grafana.settings.server.http_addr}:${toString cfg.port}";
- };
-
- services = {
- postgresql = {
- ensureUsers = [
- {
- name = "grafana";
- ensureDBOwnership = true;
- }
- ];
- ensureDatabases = [
- "grafana"
- ];
- };
-
- grafana = {
- enable = true;
-
- settings = {
- database = {
- host = "/run/postgresql";
- user = "grafana";
- name = "grafana";
- type = "postgres";
- };
-
- security = {
- secret_key = "$__file{${config.age.secrets.grafanaSecretKey.path}}";
-
- disable_initial_admin_creation = false;
- admin_password = "$__file{${config.age.secrets.grafanaInitialAdminPw.path}}";
- };
-
- server = {
- http_addr = "127.0.0.1";
- http_port = cfg.port;
- domain = cfg.fqdn;
- root_url = "https://${cfg.fqdn}";
- router_logging = cfg.debugLog;
- };
-
- smtp = lib.mkIf (!(isNull cfg.smtp)) {
- enabled = true;
- inherit (cfg.smtp) from_address from_name;
- host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
- user = cfg.smtp.username;
- password = "$__file{${cfg.smtp.passwordFile}}";
- };
- };
-
- provision = {
- dashboards.settings = lib.mkIf (cfg.dashboards != []) {
- apiVersion = 1;
- providers = [
- {
- folder = "Vhack";
- options.path = pkgs.symlinkJoin {
- name = "dashboards";
- paths = map (p: pkgs.runCommand "dashoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
- };
- allowUiUpdates = true;
- disableDeletion = true;
- }
- ];
- };
- datasources.settings = {
- apiVersion = 1;
- datasources = [
- {
- inherit (cfg) orgId;
- name = "Prometheus";
- type = "prometheus";
- url = "http://127.0.0.1:${toString config.services.prometheus.port}";
- uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
- isDefault = true;
- version = 1;
- }
- {
- inherit (cfg) orgId;
- name = "Loki";
- type = "loki";
- url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
- uid = "cd6cc53e-840c-484d-85f7-96fede324006";
- version = 1;
- }
- ];
- deleteDatasources = [
- {
- inherit (cfg) orgId;
- name = "Prometheus";
- }
- {
- inherit (cfg) orgId;
- name = "Loki";
- }
- ];
- };
- alerting = {
- contactPoints.settings = {
- apiVersion = 1;
- contactPoints = [
- {
- inherit (cfg) orgId;
- name = "grafana-default-email";
- receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [
- {
- uid = "sysadmin";
- type = "email";
- settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
- }
- ];
- }
- ];
- };
- policies.settings = {
- apiVersion = 1;
- policies = [
- {
- inherit (cfg) orgId;
- receiver = "grafana-default-email";
- group_by = [
- "grafana_folder"
- "alertname"
- ];
- group_wait = "30s";
- group_interval = "5m";
- repeat_interval = "4h";
- }
- ];
- # resetPolicies seems to happen after setting the above policies, effectively rolling back
- # any updates.
- };
- rules.settings = let
- rules = builtins.fromJSON (builtins.readFile ../rules.json);
- in {
- apiVersion = 1;
- groups = [
- {
- inherit (cfg) orgId;
- name = "SysAdmin";
- folder = "Vhack";
- interval = "10m";
- inherit rules;
- }
- ];
- # deleteRules seems to happen after creating the above rules, effectively rolling back
- # any updates.
- };
- };
- };
- };
- };
- };
-}
diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix
deleted file mode 100644
index 149e577..0000000
--- a/modules/by-name/mo/monitoring/components/loki.nix
+++ /dev/null
@@ -1,170 +0,0 @@
-{
- config,
- lib,
- pkgs,
- ...
-}: let
- cfg = config.vhack.monitoring.loki;
-in {
- options.vhack.monitoring.loki = {
- enable = lib.mkEnableOption "loki";
-
- port = lib.mkOption {
- type = lib.types.port;
- description = "Port where Loki listens to HTTP requests.";
- default = 3002;
- };
- };
-
- config = lib.mkIf cfg.enable {
- vhack = {
- persist.directories = [
- {
- directory = "/var/lib/loki";
- user = "loki";
- group = "loki";
- mode = "0700";
- }
- {
- directory = "/var/lib/fluent-bit";
- user = "fluent-bit";
- group = "fluent-bit";
- mode = "0700";
- }
- ];
- };
-
- users = {
- groups = {
- fluent-bit = {
- gid = config.vhack.constants.ids.gids.fluent-bit;
- };
- loki = {
- gid = config.vhack.constants.ids.gids.loki;
- };
- };
- users = {
- fluent-bit = {
- isSystemUser = true;
- group = "fluent-bit";
- uid = config.vhack.constants.ids.uids.fluent-bit;
- };
- loki = {
- isSystemUser = true;
- group = "loki";
- uid = config.vhack.constants.ids.uids.loki;
- };
- };
- };
-
- # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
- systemd.services.fluent-bit = {
- serviceConfig = {
- StateDirectory = "fluent-bit";
- User = "fluent-bit";
- Group = "fluent-bit";
- DynamicUser = lib.mkForce false;
- };
- };
- services = {
- fluent-bit = {
- enable = true;
- settings = {
- service = {
- flush = 1;
- log_level = "info";
- http_server = "true";
- http_listen = "127.0.0.1";
- http_port = 9080;
- grace = 30;
- };
-
- pipeline = {
- inputs = [
- {
- name = "systemd";
-
- # The asterisk appends the _SYSTEMD_UNIT to the prefix.
- tag = "systemd.*";
-
- # Read logs from this systemd journal directory.
- path = "/var/log/journal";
-
- # Database file to keep track of the journald cursor.
- db = "/var/lib/fluent-bit/systemd.db";
-
- # Start reading new entries. Skip entries already stored in journald.
- read_from_tail = true;
-
- # Max entries to lookback on start.
- max_entries = 10000;
- }
- ];
-
- outputs = [
- {
- name = "loki";
-
- match = "systemd.*";
-
- host = "localhost";
- port = config.services.loki.configuration.server.http_listen_port;
-
- labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
- job = "systemd-journal";
- inherit (config.vhack.monitoring.grafana) fqdn;
- hostname = config.networking.hostName;
- };
-
- label_keys = "$unit";
- }
- ];
- };
- };
- graceLimit = "1m";
- };
-
- loki = {
- enable = true;
- dataDir = "/var/lib/loki";
- package = pkgs.grafana-loki;
- configuration = {
- auth_enabled = false;
-
- server.http_listen_port = cfg.port;
-
- common = {
- ring = {
- instance_addr = "127.0.0.1";
- kvstore = {
- store = "inmemory";
- };
- };
-
- replication_factor = 1;
- path_prefix = "/tmp/loki";
- };
-
- schema_config = {
- configs = [
- {
- from = "2020-05-15";
- store = "tsdb";
- object_store = "filesystem";
- schema = "v13";
- index = {
- prefix = "index_";
- period = "24h";
- };
- }
- ];
- };
-
- storage_config = {
- filesystem.directory = "/tmp/loki/chunks";
- };
- };
- };
- };
- };
-}
diff --git a/modules/by-name/mo/monitoring/components/perses.nix b/modules/by-name/mo/monitoring/components/perses.nix
deleted file mode 100644
index b0f1ab6..0000000
--- a/modules/by-name/mo/monitoring/components/perses.nix
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- config,
- lib,
- pkgs,
- ...
-}: let
- cfg = config.vhack.monitoring.perses;
-in {
- options.vhack.monitoring.perses = {
- enable = lib.mkEnableOption "perses";
- };
- config = lib.mkIf cfg.enable {
- services.perses = {
- enable = true;
- };
- };
-}
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
deleted file mode 100644
index ca947d3..0000000
--- a/modules/by-name/mo/monitoring/components/prometheus.nix
+++ /dev/null
@@ -1,272 +0,0 @@
-{
- lib,
- config,
- nixLib,
- ...
-}: let
- cfg = config.vhack.monitoring.prometheus;
-
- commonLabels = {
- hostname = config.networking.hostName;
- };
-
- prom_cfg = config.services.prometheus;
-in {
- options.vhack.monitoring.prometheus = {
- enable = lib.mkEnableOption "prometheus";
-
- port = lib.mkOption {
- type = lib.types.port;
- description = "Port where Prometheus listens to HTTP requests.";
- default = 3001;
- };
-
- remoteWriteReceiver = lib.mkOption {
- example = "https://prometheus.server2.vhack.eu";
- type = lib.types.nullOr lib.types.str;
- default = null;
- description = ''
- If not null, this prometheus server will be exposed via a reverse proxy and allow
- other prometheus servers to write metrics to it.
- '';
- };
- remoteWriteTo = lib.mkOption {
- example = "https://prometheus.server3.vhack.eu";
- type = lib.types.nullOr (lib.types.submodule {
- options = {
- port = lib.mkOption {
- type = lib.types.nullOr lib.types.port;
- default = null;
- };
- url = lib.mkOption {
- type = lib.types.str;
- };
- };
- });
- default = null;
- description = ''
- If not null, this prometheus server will run in agent mode, writing all recived
- metrics to the specified remote prometheus server.
- '';
- };
-
- sources = lib.mkOption {
- type = lib.types.listOf (lib.types.submodule {
- options = {
- name = lib.mkOption {
- type = lib.types.str;
- };
- target = lib.mkOption {
- type = lib.types.str;
- description = "The target url to scrape from.";
- };
- extra = lib.mkOption {
- type = lib.types.attrsOf lib.types.anything;
- };
- };
- });
- };
- };
-
- config = lib.mkIf cfg.enable {
- vhack = {
- nginx.enable = true;
-
- persist.directories = [
- {
- directory = "/var/lib/prometheus2";
- user = "prometheus";
- group = "prometheus";
- mode = "0700";
- }
- ];
-
- monitoring.prometheus.sources = [
- {
- name = "node";
- target = "127.0.0.1:${toString prom_cfg.exporters.node.port}";
- }
- {
- name = "prometheus_internal";
- target = "127.0.0.1:${toString prom_cfg.port}";
- }
- {
- name = "systemd";
- target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}";
- }
- ];
- };
-
- services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) {
- virtualHosts.${cfg.remoteWriteReceiver} = {
- locations."/" = {
- proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write";
-
- recommendedProxySettings = true;
- proxyWebsockets = true;
- };
-
- enableACME = true;
- forceSSL = true;
- };
- };
-
- users = {
- groups = {
- node-exporter.gid = config.vhack.constants.ids.gids.node-exporter;
- };
- users = {
- node-exporter = {
- isSystemUser = true;
- group = "node-exporter";
- uid = config.vhack.constants.ids.uids.node-exporter;
- };
- };
- };
-
- services = {
- prometheus = {
- enable = true;
- inherit (cfg) port;
-
- globalConfig = {
- scrape_interval = "15s";
- };
- remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [
- {
- url = let
- port =
- if (cfg.remoteWriteTo.port != null)
- then ":${toString cfg.remoteWriteTo.port}"
- else "";
- in "${cfg.remoteWriteTo.url}${port}";
- # TODO: Needs RFC42 for prometheus <2026-07-19>
- # protobuf_message = "io.prometheus.write.v2.Request";
- }
- ];
-
- enableAgentMode = cfg.remoteWriteTo != null;
- extraFlags = lib.mkMerge [
- (
- lib.mkIf (cfg.remoteWriteReceiver != null) [
- "--web.enable-remote-write-receiver"
- # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request"
- ]
- )
- ];
-
- scrapeConfigs = builtins.map (attr:
- nixLib.warnMerge {
- job_name = attr.name;
- static_configs = [
- {
- targets = [attr.target];
- labels = commonLabels;
- }
- ];
- }
- attr.extra "prometheus scrapeConfig '${attr.name}'")
- cfg.sources;
- # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
- # job_name = "redis";
- # static_configs = [
- # {
- # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
- # }
- # ];
- # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
- # job_name = "openvpn";
- # static_configs = [
- # {
- # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
- # }
- # ];
- # })
- # ++ (lib.optional config.services.dnsmasq.enable {
- # job_name = "dnsmasq";
- # static_configs = [
- # {
- # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"];
- # labels = commonLabels;
- # }
- # ];
- # });
-
- exporters = {
- node = {
- enable = true;
- # https://github.com/prometheus/node_exporter#collectors
- enabledCollectors = [
- "arp"
- "cpu"
- "cpufreq"
- "diskstats"
- "dmi"
- "edac"
- "entropy"
- "filefd"
- "filesystem"
- "hwmon"
- "loadavg"
- "meminfo"
- "netclass"
- "netdev"
- "netstat"
- "nvme"
- "os"
- "pressure"
- "rapl"
- "schedstat"
- "stat"
- "thermal_zone"
- "time"
- "uname"
- "vmstat"
- "zfs"
-
- # Disabled by default
- "cgroups"
- "drm"
- "ethtool"
- "logind"
- "wifi"
- ];
- port = 9112;
- listenAddress = "127.0.0.1";
- };
- # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
- # enable = true;
- # port = 9119;
- # listenAddress = "127.0.0.1";
- # };
- # services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
- # enable = true;
- # port = 9121;
- # listenAddress = "127.0.0.1";
- # statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
- # };
- # dnsmasq = lib.mkIf config.services.dnsmasq.enable {
- # enable = true;
- # port = 9211;
- # listenAddress = "127.0.0.1";
- # };
- systemd = {
- enable = true;
- port = 9116;
- listenAddress = "127.0.0.1";
- extraFlags = [
- "--log.level=info"
- "--web.disable-exporter-metrics"
- # "--systemd.collector.enable-resolved"
- "--systemd.collector.enable-restart-count"
- # "--systemd.collector.enable-file-descriptor-size"
- "--systemd.collector.enable-ip-accounting"
- "--systemd.collector.unit-include=\".+\""
- "--systemd.collector.unit-exclude=\".+\\\\.(device)\""
- ];
- };
- };
- };
- };
- };
-}
diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix
deleted file mode 100644
index adb515e..0000000
--- a/modules/by-name/mo/monitoring/components/scrutiny.nix
+++ /dev/null
@@ -1,100 +0,0 @@
-{
- config,
- pkgs,
- lib,
- ...
-}: let
- cfg = config.vhack.monitoring.scrutiny;
-in {
- options.vhack.monitoring.scrutiny = {
- enable = lib.mkEnableOption "scrutiny service";
- };
-
- # We don't currently _care_ about the health of the hardware, because hetzner does
- # this for us (nor can we care, because it's not exposed in the VPS)
- config = lib.mkIf false {
- vhack.monitoring = {
- prometheus.sources = [
- {
- name = "scrutiny";
- extra.metrics_path = "/api/metrics";
- target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
- }
- {
- name = "smartctl";
- target = "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}";
- }
- ];
- grafana.dashboards = [
- ../dashboards/Health.json
- ];
- };
-
- users = {
- groups = {
- smartctl-exporter-access.gid = config.vhack.constants.ids.gids.smartctl-exporter-access;
- };
- users = {
- smartctl-exporter-access = {
- isSystemUser = true;
- group = "smartctl-exporter-access";
- uid = config.vhack.constants.ids.uids.smartctl-exporter-access;
- };
- };
- };
-
- services = {
- scrutiny = {
- enable = true;
-
- openFirewall = false;
-
- # This src includes Prometheus metrics exporter.
- package = pkgs.scrutiny.overrideAttrs {
- src = pkgs.fetchFromGitHub {
- owner = "ibizaman";
- repo = "scrutiny";
- rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
- hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
- };
-
- vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
- };
-
- settings = {
- web = {
- metrics.enabled = true; # Enables Prometheus exporter
- listenHost = "127.0.0.1";
- };
- };
-
- collector = {
- enable = true;
- };
- };
- prometheus = {
- exporters.smartctl = {
- enable = true;
- port = 9115;
- listenAddress = "127.0.0.1";
- };
- };
- };
-
- # nixpkgs.overlays = [
- # (final: prev: {
- # prometheus-systemd-exporter = prev.prometheus-systemd-exporter.overrideAttrs {
- # src = final.fetchFromGitHub {
- # owner = "ibizaman";
- # repo = prev.prometheus-systemd-exporter.pname;
- # # rev = "v${prev.prometheus-systemd-exporter.version}";
- # rev = "next_timer";
- # sha256 = "sha256-jzkh/616tsJbNxFtZ0xbdBQc16TMIYr9QOkPaeQw8xA=";
- # };
- #
- # vendorHash = "sha256-4hsQ1417jLNOAqGkfCkzrmEtYR4YLLW2j0CiJtPg6GI=";
- # };
- # })
- # ];
- };
-}
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-19 22:46:27 +0000