aboutsummaryrefslogtreecommitdiffstats
path: root/modules/by-name/mo/monitoring/components
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--modules/by-name/mo/monitoring/components/grafana.nix296
-rw-r--r--modules/by-name/mo/monitoring/components/loki.nix122
-rw-r--r--modules/by-name/mo/monitoring/components/perses.nix17
-rw-r--r--modules/by-name/mo/monitoring/components/prometheus.nix192
-rw-r--r--modules/by-name/mo/monitoring/components/scrutiny.nix106
5 files changed, 733 insertions, 0 deletions
diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix
new file mode 100644
index 0000000..5459878
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/grafana.nix
@@ -0,0 +1,296 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}: let
+ cfg = config.vhack.monitoring.grafana;
+in {
+ options.vhack.monitoring.grafana = {
+ enable = lib.mkEnableOption "grafana";
+
+ orgId = lib.mkOption {
+ type = lib.types.int;
+ description = "Org ID where all vhack related config will be stored.";
+ default = 1;
+ };
+
+ contactPoints = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ description = "List of email addresses to send alerts to";
+ default = [];
+ };
+
+ dashboards = lib.mkOption {
+ type = lib.types.listOf lib.types.path;
+ description = "dashboards to provision under 'Vhack' folder.";
+ default = [];
+ };
+
+ fqdn = lib.mkOption {
+ type = lib.types.str;
+ description = "Domain under which Grafana will be served.";
+ example = "grafana.example.org";
+ };
+
+ adminPassword = lib.mkOption {
+ type = lib.types.path;
+ description = "The age-encrypted initial admin password; passed to agenix";
+ };
+
+ secretKey = lib.mkOption {
+ type = lib.types.path;
+ description = "The age-encrypted secret key, used for signing; passed to agenix";
+ };
+
+ port = lib.mkOption {
+ type = lib.types.port;
+ description = "Port where Grafana listens to HTTP requests.";
+ default = 3000;
+ };
+
+ debugLog = lib.mkOption {
+ type = lib.types.bool;
+ description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
+ default = false;
+ example = true;
+ };
+
+ smtp = lib.mkOption {
+ description = "SMTP options.";
+ default = null;
+ type = lib.types.nullOr (
+ lib.types.submodule {
+ options = {
+ from_address = lib.mkOption {
+ type = lib.types.str;
+ description = "SMTP address from which the emails originate.";
+ example = "vaultwarden@mydomain.com";
+ };
+ from_name = lib.mkOption {
+ type = lib.types.str;
+ description = "SMTP name from which the emails originate.";
+ default = "Grafana";
+ };
+ host = lib.mkOption {
+ type = lib.types.str;
+ description = "SMTP host to send the emails to.";
+ };
+ port = lib.mkOption {
+ type = lib.types.port;
+ description = "SMTP port to send the emails to.";
+ default = 25;
+ };
+ username = lib.mkOption {
+ type = lib.types.str;
+ description = "Username to connect to the SMTP host.";
+ };
+ passwordFile = lib.mkOption {
+ type = lib.types.str;
+ description = "File containing the password to connect to the SMTP host.";
+ };
+ };
+ }
+ );
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ assertions = [
+ {
+ assertion = builtins.length cfg.contactPoints > 0;
+ message = "Must have at least one contact point for alerting";
+ }
+ ];
+
+ age.secrets = {
+ grafanaInitialAdminPw = {
+ file = cfg.adminPassword;
+ mode = "0700";
+ owner = "grafana";
+ group = "grafana";
+ };
+ grafanaSecretKey = {
+ file = cfg.secretKey;
+ mode = "0700";
+ owner = "grafana";
+ group = "grafana";
+ };
+ };
+
+ vhack = {
+ postgresql.enable = true;
+ monitoring.grafana.dashboards = [
+ ../dashboards/Errors.json
+ ../dashboards/Performance.json
+ ../dashboards/Scraping_Jobs.json
+ ];
+ };
+
+ services = {
+ nginx = {
+ virtualHosts.${cfg.fqdn} = {
+ locations."/" = {
+ proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
+ proxyWebsockets = true;
+ extraConfig = ''
+ proxy_set_header Host $host;
+ '';
+ };
+
+ forceSSL = true;
+ enableACME = true;
+ };
+ };
+
+ postgresql = {
+ ensureUsers = [
+ {
+ name = "grafana";
+ ensureDBOwnership = true;
+ }
+ ];
+ ensureDatabases = [
+ "grafana"
+ ];
+ };
+
+ grafana = {
+ enable = true;
+
+ settings = {
+ database = {
+ host = "/run/postgresql";
+ user = "grafana";
+ name = "grafana";
+ type = "postgres";
+ };
+
+ security = {
+ secret_key = "$__file{${config.age.secrets.grafanaSecretKey.path}}";
+
+ disable_initial_admin_creation = false;
+ admin_password = "$__file{${config.age.secrets.grafanaInitialAdminPw.path}}";
+ };
+
+ server = {
+ http_addr = "127.0.0.1";
+ http_port = cfg.port;
+ domain = cfg.fqdn;
+ root_url = "https://${cfg.fqdn}";
+ router_logging = cfg.debugLog;
+ };
+
+ smtp = lib.mkIf (!(isNull cfg.smtp)) {
+ enabled = true;
+ inherit (cfg.smtp) from_address from_name;
+ host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
+ user = cfg.smtp.username;
+ password = "$__file{${cfg.smtp.passwordFile}}";
+ };
+ };
+
+ provision = {
+ dashboards.settings = lib.mkIf (cfg.dashboards != []) {
+ apiVersion = 1;
+ providers = [
+ {
+ folder = "Vhack";
+ options.path = pkgs.symlinkJoin {
+ name = "dashboards";
+ paths = map (p: pkgs.runCommand "davhackoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
+ };
+ allowUiUpdates = true;
+ disableDeletion = true;
+ }
+ ];
+ };
+ datasources.settings = {
+ apiVersion = 1;
+ datasources = [
+ {
+ inherit (cfg) orgId;
+ name = "Prometheus";
+ type = "prometheus";
+ url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+ uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
+ isDefault = true;
+ version = 1;
+ }
+ {
+ inherit (cfg) orgId;
+ name = "Loki";
+ type = "loki";
+ url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+ uid = "cd6cc53e-840c-484d-85f7-96fede324006";
+ version = 1;
+ }
+ ];
+ deleteDatasources = [
+ {
+ inherit (cfg) orgId;
+ name = "Prometheus";
+ }
+ {
+ inherit (cfg) orgId;
+ name = "Loki";
+ }
+ ];
+ };
+ alerting = {
+ contactPoints.settings = {
+ apiVersion = 1;
+ contactPoints = [
+ {
+ inherit (cfg) orgId;
+ name = "grafana-default-email";
+ receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [
+ {
+ uid = "sysadmin";
+ type = "email";
+ settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
+ }
+ ];
+ }
+ ];
+ };
+ policies.settings = {
+ apiVersion = 1;
+ policies = [
+ {
+ inherit (cfg) orgId;
+ receiver = "grafana-default-email";
+ group_by = [
+ "grafana_folder"
+ "alertname"
+ ];
+ group_wait = "30s";
+ group_interval = "5m";
+ repeat_interval = "4h";
+ }
+ ];
+ # resetPolicies seems to happen after setting the above policies, effectively rolling back
+ # any updates.
+ };
+ rules.settings = let
+ rules = builtins.fromJSON (builtins.readFile ../rules.json);
+ in {
+ apiVersion = 1;
+ groups = [
+ {
+ inherit (cfg) orgId;
+ name = "SysAdmin";
+ folder = "Vhack";
+ interval = "10m";
+ inherit rules;
+ }
+ ];
+ # deleteRules seems to happen after creating the above rules, effectively rolling back
+ # any updates.
+ };
+ };
+ };
+ };
+ };
+ };
+}
diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix
new file mode 100644
index 0000000..9a285c5
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/loki.nix
@@ -0,0 +1,122 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}: let
+ cfg = config.vhack.monitoring.loki;
+in {
+ options.vhack.monitoring.loki = {
+ enable = lib.mkEnableOption "loki";
+
+ port = lib.mkOption {
+ type = lib.types.port;
+ description = "Port where Loki listens to HTTP requests.";
+ default = 3002;
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ services = {
+ loki = {
+ enable = true;
+ dataDir = "/var/lib/loki";
+ package = pkgs.grafana-loki;
+ configuration = {
+ auth_enabled = false;
+
+ server.http_listen_port = cfg.port;
+
+ common = {
+ ring = {
+ instance_addr = "127.0.0.1";
+ kvstore = {
+ store = "inmemory";
+ };
+ };
+
+ replication_factor = 1;
+ path_prefix = "/tmp/loki";
+ };
+
+ schema_config = {
+ configs = [
+ {
+ from = "2020-05-15";
+ store = "tsdb";
+ object_store = "filesystem";
+ schema = "v13";
+ index = {
+ prefix = "index_";
+ period = "24h";
+ };
+ }
+ ];
+ };
+
+ storage_config = {
+ filesystem.directory = "/tmp/loki/chunks";
+ };
+ };
+ };
+
+ # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+ fluent-bit = {
+ enable = true;
+ settings = {
+ service = {
+ flush = 1;
+ log_level = "info";
+ http_server = "true";
+ http_listen = "127.0.0.1";
+ http_port = 9080;
+ grace = 30;
+ };
+
+ pipeline = {
+ inputs = [
+ {
+ name = "systemd";
+
+ # The asterisk appends the _SYSTEMD_UNIT to the prefix.
+ tag = "systemd.*";
+
+ # Read logs from this systemd journal directory.
+ path = "/var/log/journal";
+
+ # Database file to keep track of the journald cursor.
+ db = "/var/fluent-bit/systemd.db";
+
+ # Start reading new entries. Skip entries already stored in journald.
+ read_from_tail = true;
+
+ # Max entries to lookback on start.
+ max_entries = 10000;
+ }
+ ];
+
+ outputs = [
+ {
+ name = "loki";
+
+ match = "systemd.*";
+
+ host = "localhost";
+ port = config.services.loki.configuration.server.http_listen_port;
+
+ labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
+ job = "systemd-journal";
+ inherit (config.vhack.monitoring.grafana) fqdn;
+ hostname = config.networking.hostName;
+ };
+
+ label_keys = "$unit";
+ }
+ ];
+ };
+ };
+ graceLimit = "1m";
+ };
+ };
+ };
+}
diff --git a/modules/by-name/mo/monitoring/components/perses.nix b/modules/by-name/mo/monitoring/components/perses.nix
new file mode 100644
index 0000000..b0f1ab6
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/perses.nix
@@ -0,0 +1,17 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}: let
+ cfg = config.vhack.monitoring.perses;
+in {
+ options.vhack.monitoring.perses = {
+ enable = lib.mkEnableOption "perses";
+ };
+ config = lib.mkIf cfg.enable {
+ services.perses = {
+ enable = true;
+ };
+ };
+}
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
new file mode 100644
index 0000000..468ece3
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -0,0 +1,192 @@
+{
+ lib,
+ config,
+ nixLib,
+ ...
+}: let
+ cfg = config.vhack.monitoring.prometheus;
+
+ commonLabels = {
+ hostname = config.networking.hostName;
+ inherit (config.vhack.monitoring.grafana) fqdn;
+ };
+
+ prom_cfg = config.services.prometheus;
+in {
+ options.vhack.monitoring.prometheus = {
+ enable = lib.mkEnableOption "prometheus";
+
+ port = lib.mkOption {
+ type = lib.types.port;
+ description = "Port where Prometheus listens to HTTP requests.";
+ default = 3001;
+ };
+
+ sources = lib.mkOption {
+ type = lib.types.listOf (lib.types.submodule {
+ options = {
+ name = lib.mkOption {
+ type = lib.types.str;
+ };
+ target = lib.mkOption {
+ type = lib.types.str;
+ };
+ extra = lib.mkOption {
+ type = lib.types.attrsOf lib.types.anything;
+ };
+ };
+ });
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ vhack = {
+ nginx.enable = true;
+ monitoring.prometheus.sources = [
+ {
+ name = "node";
+ target = "127.0.0.1:${toString prom_cfg.exporters.node.port}";
+ }
+ {
+ name = "smartctl";
+ target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}";
+ }
+ {
+ name = "prometheus_internal";
+ target = "127.0.0.1:${toString prom_cfg.port}";
+ }
+ {
+ name = "systemd";
+ target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}";
+ }
+ {
+ name = "netdata";
+ target = "127.0.0.1:19999";
+ extra = {
+ metrics_path = "/api/v1/allmetrics";
+ params.format = ["prometheus"];
+ honor_labels = true;
+ };
+ }
+ ];
+ };
+
+ services = {
+ prometheus = {
+ enable = true;
+ inherit (cfg) port;
+
+ globalConfig = {
+ scrape_interval = "15s";
+ };
+
+ scrapeConfigs = builtins.map (attr:
+ nixLib.warnMerge {
+ job_name = attr.name;
+ static_configs = [
+ {
+ targets = [attr.target];
+ labels = commonLabels;
+ }
+ ];
+ }
+ attr.extra "prometheus scrapeConfig")
+ cfg.sources;
+ # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
+ # job_name = "redis";
+ # static_configs = [
+ # {
+ # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
+ # }
+ # ];
+ # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
+ # job_name = "openvpn";
+ # static_configs = [
+ # {
+ # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
+ # }
+ # ];
+ # })
+ # ++ (lib.optional config.services.dnsmasq.enable {
+ # job_name = "dnsmasq";
+ # static_configs = [
+ # {
+ # targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"];
+ # labels = commonLabels;
+ # }
+ # ];
+ # });
+
+ exporters = {
+ node = {
+ enable = true;
+ # https://github.com/prometheus/node_exporter#collectors
+ enabledCollectors = [
+ "arp"
+ "cpu"
+ "cpufreq"
+ "diskstats"
+ "dmi"
+ "edac"
+ "entropy"
+ "filefd"
+ "filesystem"
+ "hwmon"
+ "loadavg"
+ "meminfo"
+ "netclass"
+ "netdev"
+ "netstat"
+ "nvme"
+ "os"
+ "pressure"
+ "rapl"
+ "schedstat"
+ "stat"
+ "thermal_zone"
+ "time"
+ "uname"
+ "vmstat"
+ "zfs"
+
+ # Disabled by default
+ "cgroups"
+ "drm"
+ "ethtool"
+ "logind"
+ "wifi"
+ ];
+ port = 9112;
+ listenAddress = "127.0.0.1";
+ };
+ smartctl = {
+ enable = true;
+ port = 9115;
+ listenAddress = "127.0.0.1";
+ };
+ # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
+ # enable = true;
+ # port = 9119;
+ # listenAddress = "127.0.0.1";
+ # };
+ # services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
+ # enable = true;
+ # port = 9121;
+ # listenAddress = "127.0.0.1";
+ # statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
+ # };
+ # dnsmasq = lib.mkIf config.services.dnsmasq.enable {
+ # enable = true;
+ # port = 9211;
+ # listenAddress = "127.0.0.1";
+ # };
+ systemd = {
+ enable = true;
+ port = 9116;
+ listenAddress = "127.0.0.1";
+ };
+ };
+ };
+ };
+ };
+}
diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix
new file mode 100644
index 0000000..31871cd
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/scrutiny.nix
@@ -0,0 +1,106 @@
+{
+ config,
+ pkgs,
+ lib,
+ ...
+}: let
+ cfg = config.vhack.monitoring.scrutiny;
+in {
+ options.vhack.monitoring.scrutiny = {
+ enable = lib.mkEnableOption "scrutiny service";
+
+ fqdn = lib.mkOption {
+ type = lib.types.nullOr lib.types.str;
+ description = ''
+ If a string, this will be the domain under which the scrutiny web interface will be servced.
+
+ If null, the web interface will not be served and only the prometheus metrics will be accessible.
+ '';
+ };
+ };
+
+ config = {
+ vhack.monitoring = {
+ prometheus.sources = [
+ {
+ name = "scrutiny";
+ extra.metrics_path = "/api/metrics";
+ target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
+ }
+ ];
+ grafana.dashboards = [
+ ../dashboards/Health.json
+ ];
+ };
+
+ services = {
+ scrutiny = {
+ enable = true;
+
+ openFirewall = false;
+
+ # This src includes Prometheus metrics exporter.
+ package = pkgs.scrutiny.overrideAttrs {
+ src = pkgs.fetchFromGitHub {
+ owner = "ibizaman";
+ repo = "scrutiny";
+ rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
+ hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
+ };
+
+ vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
+ };
+
+ settings = {
+ web = {
+ metrics.enabled = true; # Enables Prometheus exporter
+ listenHost = "127.0.0.1";
+ };
+ };
+
+ collector = {
+ enable = true;
+ };
+ };
+
+ # nginx.statusPage = lib.mkDefault config.services.nginx.enable;
+ nginx = lib.mkIf (cfg.fqdn != null) {
+ virtualHosts."${cfg.fqdn}" = {
+ locations."/".proxyPass = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
+
+ enableACME = true;
+ forceSSL = true;
+ };
+ };
+
+ netdata = {
+ enable = true;
+ config = {
+ # web.mode = "none";
+ # web."bind to" = "127.0.0.1:19999";
+ global = {
+ "debug log" = "syslog";
+ "access log" = "syslog";
+ "error log" = "syslog";
+ };
+ };
+ };
+ };
+
+ # nixpkgs.overlays = [
+ # (final: prev: {
+ # prometheus-systemd-exporter = prev.prometheus-systemd-exporter.overrideAttrs {
+ # src = final.fetchFromGitHub {
+ # owner = "ibizaman";
+ # repo = prev.prometheus-systemd-exporter.pname;
+ # # rev = "v${prev.prometheus-systemd-exporter.version}";
+ # rev = "next_timer";
+ # sha256 = "sha256-jzkh/616tsJbNxFtZ0xbdBQc16TMIYr9QOkPaeQw8xA=";
+ # };
+ #
+ # vendorHash = "sha256-4hsQ1417jLNOAqGkfCkzrmEtYR4YLLW2j0CiJtPg6GI=";
+ # };
+ # })
+ # ];
+ };
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-21 05:54:00 +0000