{modules,test}/monitoring: Init basic version

5 files changed, 733 insertions, 0 deletions

diff --git a/modules/by-name/mo/monitoring/components/grafana.nix b/modules/by-name/mo/monitoring/components/grafana.nix
new file mode 100644
index 0000000..5459878
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/grafana.nix
@@ -0,0 +1,296 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  cfg = config.vhack.monitoring.grafana;
+in {
+  options.vhack.monitoring.grafana = {
+    enable = lib.mkEnableOption "grafana";
+
+    orgId = lib.mkOption {
+      type = lib.types.int;
+      description = "Org ID where all vhack related config will be stored.";
+      default = 1;
+    };
+
+    contactPoints = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "List of email addresses to send alerts to";
+      default = [];
+    };
+
+    dashboards = lib.mkOption {
+      type = lib.types.listOf lib.types.path;
+      description = "dashboards to provision under 'Vhack' folder.";
+      default = [];
+    };
+
+    fqdn = lib.mkOption {
+      type = lib.types.str;
+      description = "Domain under which Grafana will be served.";
+      example = "grafana.example.org";
+    };
+
+    adminPassword = lib.mkOption {
+      type = lib.types.path;
+      description = "The age-encrypted initial admin password; passed to agenix";
+    };
+
+    secretKey = lib.mkOption {
+      type = lib.types.path;
+      description = "The age-encrypted secret key, used for signing; passed to agenix";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      description = "Port where Grafana listens to HTTP requests.";
+      default = 3000;
+    };
+
+    debugLog = lib.mkOption {
+      type = lib.types.bool;
+      description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
+      default = false;
+      example = true;
+    };
+
+    smtp = lib.mkOption {
+      description = "SMTP options.";
+      default = null;
+      type = lib.types.nullOr (
+        lib.types.submodule {
+          options = {
+            from_address = lib.mkOption {
+              type = lib.types.str;
+              description = "SMTP address from which the emails originate.";
+              example = "vaultwarden@mydomain.com";
+            };
+            from_name = lib.mkOption {
+              type = lib.types.str;
+              description = "SMTP name from which the emails originate.";
+              default = "Grafana";
+            };
+            host = lib.mkOption {
+              type = lib.types.str;
+              description = "SMTP host to send the emails to.";
+            };
+            port = lib.mkOption {
+              type = lib.types.port;
+              description = "SMTP port to send the emails to.";
+              default = 25;
+            };
+            username = lib.mkOption {
+              type = lib.types.str;
+              description = "Username to connect to the SMTP host.";
+            };
+            passwordFile = lib.mkOption {
+              type = lib.types.str;
+              description = "File containing the password to connect to the SMTP host.";
+            };
+          };
+        }
+      );
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = builtins.length cfg.contactPoints > 0;
+        message = "Must have at least one contact point for alerting";
+      }
+    ];
+
+    age.secrets = {
+      grafanaInitialAdminPw = {
+        file = cfg.adminPassword;
+        mode = "0700";
+        owner = "grafana";
+        group = "grafana";
+      };
+      grafanaSecretKey = {
+        file = cfg.secretKey;
+        mode = "0700";
+        owner = "grafana";
+        group = "grafana";
+      };
+    };
+
+    vhack = {
+      postgresql.enable = true;
+      monitoring.grafana.dashboards = [
+        ../dashboards/Errors.json
+        ../dashboards/Performance.json
+        ../dashboards/Scraping_Jobs.json
+      ];
+    };
+
+    services = {
+      nginx = {
+        virtualHosts.${cfg.fqdn} = {
+          locations."/" = {
+            proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
+            proxyWebsockets = true;
+            extraConfig = ''
+              proxy_set_header Host $host;
+            '';
+          };
+
+          forceSSL = true;
+          enableACME = true;
+        };
+      };
+
+      postgresql = {
+        ensureUsers = [
+          {
+            name = "grafana";
+            ensureDBOwnership = true;
+          }
+        ];
+        ensureDatabases = [
+          "grafana"
+        ];
+      };
+
+      grafana = {
+        enable = true;
+
+        settings = {
+          database = {
+            host = "/run/postgresql";
+            user = "grafana";
+            name = "grafana";
+            type = "postgres";
+          };
+
+          security = {
+            secret_key = "$__file{${config.age.secrets.grafanaSecretKey.path}}";
+
+            disable_initial_admin_creation = false;
+            admin_password = "$__file{${config.age.secrets.grafanaInitialAdminPw.path}}";
+          };
+
+          server = {
+            http_addr = "127.0.0.1";
+            http_port = cfg.port;
+            domain = cfg.fqdn;
+            root_url = "https://${cfg.fqdn}";
+            router_logging = cfg.debugLog;
+          };
+
+          smtp = lib.mkIf (!(isNull cfg.smtp)) {
+            enabled = true;
+            inherit (cfg.smtp) from_address from_name;
+            host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
+            user = cfg.smtp.username;
+            password = "$__file{${cfg.smtp.passwordFile}}";
+          };
+        };
+
+        provision = {
+          dashboards.settings = lib.mkIf (cfg.dashboards != []) {
+            apiVersion = 1;
+            providers = [
+              {
+                folder = "Vhack";
+                options.path = pkgs.symlinkJoin {
+                  name = "dashboards";
+                  paths = map (p: pkgs.runCommand "davhackoard" {} "mkdir $out; cp ${p} $out") cfg.dashboards;
+                };
+                allowUiUpdates = true;
+                disableDeletion = true;
+              }
+            ];
+          };
+          datasources.settings = {
+            apiVersion = 1;
+            datasources = [
+              {
+                inherit (cfg) orgId;
+                name = "Prometheus";
+                type = "prometheus";
+                url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+                uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
+                isDefault = true;
+                version = 1;
+              }
+              {
+                inherit (cfg) orgId;
+                name = "Loki";
+                type = "loki";
+                url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+                uid = "cd6cc53e-840c-484d-85f7-96fede324006";
+                version = 1;
+              }
+            ];
+            deleteDatasources = [
+              {
+                inherit (cfg) orgId;
+                name = "Prometheus";
+              }
+              {
+                inherit (cfg) orgId;
+                name = "Loki";
+              }
+            ];
+          };
+          alerting = {
+            contactPoints.settings = {
+              apiVersion = 1;
+              contactPoints = [
+                {
+                  inherit (cfg) orgId;
+                  name = "grafana-default-email";
+                  receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [
+                    {
+                      uid = "sysadmin";
+                      type = "email";
+                      settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
+                    }
+                  ];
+                }
+              ];
+            };
+            policies.settings = {
+              apiVersion = 1;
+              policies = [
+                {
+                  inherit (cfg) orgId;
+                  receiver = "grafana-default-email";
+                  group_by = [
+                    "grafana_folder"
+                    "alertname"
+                  ];
+                  group_wait = "30s";
+                  group_interval = "5m";
+                  repeat_interval = "4h";
+                }
+              ];
+              # resetPolicies seems to happen after setting the above policies, effectively rolling back
+              # any updates.
+            };
+            rules.settings = let
+              rules = builtins.fromJSON (builtins.readFile ../rules.json);
+            in {
+              apiVersion = 1;
+              groups = [
+                {
+                  inherit (cfg) orgId;
+                  name = "SysAdmin";
+                  folder = "Vhack";
+                  interval = "10m";
+                  inherit rules;
+                }
+              ];
+              # deleteRules seems to happen after creating the above rules, effectively rolling back
+              # any updates.
+            };
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/modules/by-name/mo/monitoring/components/loki.nix b/modules/by-name/mo/monitoring/components/loki.nix
new file mode 100644
index 0000000..9a285c5
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/loki.nix
@@ -0,0 +1,122 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  cfg = config.vhack.monitoring.loki;
+in {
+  options.vhack.monitoring.loki = {
+    enable = lib.mkEnableOption "loki";
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      description = "Port where Loki listens to HTTP requests.";
+      default = 3002;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services = {
+      loki = {
+        enable = true;
+        dataDir = "/var/lib/loki";
+        package = pkgs.grafana-loki;
+        configuration = {
+          auth_enabled = false;
+
+          server.http_listen_port = cfg.port;
+
+          common = {
+            ring = {
+              instance_addr = "127.0.0.1";
+              kvstore = {
+                store = "inmemory";
+              };
+            };
+
+            replication_factor = 1;
+            path_prefix = "/tmp/loki";
+          };
+
+          schema_config = {
+            configs = [
+              {
+                from = "2020-05-15";
+                store = "tsdb";
+                object_store = "filesystem";
+                schema = "v13";
+                index = {
+                  prefix = "index_";
+                  period = "24h";
+                };
+              }
+            ];
+          };
+
+          storage_config = {
+            filesystem.directory = "/tmp/loki/chunks";
+          };
+        };
+      };
+
+      # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+      fluent-bit = {
+        enable = true;
+        settings = {
+          service = {
+            flush = 1;
+            log_level = "info";
+            http_server = "true";
+            http_listen = "127.0.0.1";
+            http_port = 9080;
+            grace = 30;
+          };
+
+          pipeline = {
+            inputs = [
+              {
+                name = "systemd";
+
+                # The asterisk appends the _SYSTEMD_UNIT to the prefix.
+                tag = "systemd.*";
+
+                # Read logs from this systemd journal directory.
+                path = "/var/log/journal";
+
+                # Database file to keep track of the journald cursor.
+                db = "/var/fluent-bit/systemd.db";
+
+                # Start reading new entries. Skip entries already stored in journald.
+                read_from_tail = true;
+
+                # Max entries to lookback on start.
+                max_entries = 10000;
+              }
+            ];
+
+            outputs = [
+              {
+                name = "loki";
+
+                match = "systemd.*";
+
+                host = "localhost";
+                port = config.services.loki.configuration.server.http_listen_port;
+
+                labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
+                  job = "systemd-journal";
+                  inherit (config.vhack.monitoring.grafana) fqdn;
+                  hostname = config.networking.hostName;
+                };
+
+                label_keys = "$unit";
+              }
+            ];
+          };
+        };
+        graceLimit = "1m";
+      };
+    };
+  };
+}
diff --git a/modules/by-name/mo/monitoring/components/perses.nix b/modules/by-name/mo/monitoring/components/perses.nix
new file mode 100644
index 0000000..b0f1ab6
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/perses.nix
@@ -0,0 +1,17 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  cfg = config.vhack.monitoring.perses;
+in {
+  options.vhack.monitoring.perses = {
+    enable = lib.mkEnableOption "perses";
+  };
+  config = lib.mkIf cfg.enable {
+    services.perses = {
+      enable = true;
+    };
+  };
+}
diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
new file mode 100644
index 0000000..468ece3
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -0,0 +1,192 @@
+{
+  lib,
+  config,
+  nixLib,
+  ...
+}: let
+  cfg = config.vhack.monitoring.prometheus;
+
+  commonLabels = {
+    hostname = config.networking.hostName;
+    inherit (config.vhack.monitoring.grafana) fqdn;
+  };
+
+  prom_cfg = config.services.prometheus;
+in {
+  options.vhack.monitoring.prometheus = {
+    enable = lib.mkEnableOption "prometheus";
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      description = "Port where Prometheus listens to HTTP requests.";
+      default = 3001;
+    };
+
+    sources = lib.mkOption {
+      type = lib.types.listOf (lib.types.submodule {
+        options = {
+          name = lib.mkOption {
+            type = lib.types.str;
+          };
+          target = lib.mkOption {
+            type = lib.types.str;
+          };
+          extra = lib.mkOption {
+            type = lib.types.attrsOf lib.types.anything;
+          };
+        };
+      });
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    vhack = {
+      nginx.enable = true;
+      monitoring.prometheus.sources = [
+        {
+          name = "node";
+          target = "127.0.0.1:${toString prom_cfg.exporters.node.port}";
+        }
+        {
+          name = "smartctl";
+          target = "127.0.0.1:${toString prom_cfg.exporters.smartctl.port}";
+        }
+        {
+          name = "prometheus_internal";
+          target = "127.0.0.1:${toString prom_cfg.port}";
+        }
+        {
+          name = "systemd";
+          target = "127.0.0.1:${toString prom_cfg.exporters.systemd.port}";
+        }
+        {
+          name = "netdata";
+          target = "127.0.0.1:19999";
+          extra = {
+            metrics_path = "/api/v1/allmetrics";
+            params.format = ["prometheus"];
+            honor_labels = true;
+          };
+        }
+      ];
+    };
+
+    services = {
+      prometheus = {
+        enable = true;
+        inherit (cfg) port;
+
+        globalConfig = {
+          scrape_interval = "15s";
+        };
+
+        scrapeConfigs = builtins.map (attr:
+          nixLib.warnMerge {
+            job_name = attr.name;
+            static_configs = [
+              {
+                targets = [attr.target];
+                labels = commonLabels;
+              }
+            ];
+          }
+          attr.extra "prometheus scrapeConfig")
+        cfg.sources;
+        # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
+        #     job_name = "redis";
+        #     static_configs = [
+        #       {
+        #         targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
+        #       }
+        #     ];
+        # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
+        #     job_name = "openvpn";
+        #     static_configs = [
+        #       {
+        #         targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
+        #       }
+        #     ];
+        # })
+        # ++ (lib.optional config.services.dnsmasq.enable {
+        #   job_name = "dnsmasq";
+        #   static_configs = [
+        #     {
+        #       targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"];
+        #       labels = commonLabels;
+        #     }
+        #   ];
+        # });
+
+        exporters = {
+          node = {
+            enable = true;
+            # https://github.com/prometheus/node_exporter#collectors
+            enabledCollectors = [
+              "arp"
+              "cpu"
+              "cpufreq"
+              "diskstats"
+              "dmi"
+              "edac"
+              "entropy"
+              "filefd"
+              "filesystem"
+              "hwmon"
+              "loadavg"
+              "meminfo"
+              "netclass"
+              "netdev"
+              "netstat"
+              "nvme"
+              "os"
+              "pressure"
+              "rapl"
+              "schedstat"
+              "stat"
+              "thermal_zone"
+              "time"
+              "uname"
+              "vmstat"
+              "zfs"
+
+              # Disabled by default
+              "cgroups"
+              "drm"
+              "ethtool"
+              "logind"
+              "wifi"
+            ];
+            port = 9112;
+            listenAddress = "127.0.0.1";
+          };
+          smartctl = {
+            enable = true;
+            port = 9115;
+            listenAddress = "127.0.0.1";
+          };
+          # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
+          #   enable = true;
+          #   port = 9119;
+          #   listenAddress = "127.0.0.1";
+          # };
+          # services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
+          #   enable = true;
+          #   port = 9121;
+          #   listenAddress = "127.0.0.1";
+          #   statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
+          # };
+          # dnsmasq = lib.mkIf config.services.dnsmasq.enable {
+          #   enable = true;
+          #   port = 9211;
+          #   listenAddress = "127.0.0.1";
+          # };
+          systemd = {
+            enable = true;
+            port = 9116;
+            listenAddress = "127.0.0.1";
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/modules/by-name/mo/monitoring/components/scrutiny.nix b/modules/by-name/mo/monitoring/components/scrutiny.nix
new file mode 100644
index 0000000..31871cd
--- /dev/null
+++ b/modules/by-name/mo/monitoring/components/scrutiny.nix
@@ -0,0 +1,106 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  cfg = config.vhack.monitoring.scrutiny;
+in {
+  options.vhack.monitoring.scrutiny = {
+    enable = lib.mkEnableOption "scrutiny service";
+
+    fqdn = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        If a string, this will be the domain under which the scrutiny web interface will be servced.
+
+        If null, the web interface will not be served and only the prometheus metrics will be accessible.
+      '';
+    };
+  };
+
+  config = {
+    vhack.monitoring = {
+      prometheus.sources = [
+        {
+          name = "scrutiny";
+          extra.metrics_path = "/api/metrics";
+          target = "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
+        }
+      ];
+      grafana.dashboards = [
+        ../dashboards/Health.json
+      ];
+    };
+
+    services = {
+      scrutiny = {
+        enable = true;
+
+        openFirewall = false;
+
+        # This src includes Prometheus metrics exporter.
+        package = pkgs.scrutiny.overrideAttrs {
+          src = pkgs.fetchFromGitHub {
+            owner = "ibizaman";
+            repo = "scrutiny";
+            rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
+            hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
+          };
+
+          vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
+        };
+
+        settings = {
+          web = {
+            metrics.enabled = true; # Enables Prometheus exporter
+            listenHost = "127.0.0.1";
+          };
+        };
+
+        collector = {
+          enable = true;
+        };
+      };
+
+      # nginx.statusPage = lib.mkDefault config.services.nginx.enable;
+      nginx = lib.mkIf (cfg.fqdn != null) {
+        virtualHosts."${cfg.fqdn}" = {
+          locations."/".proxyPass = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
+
+          enableACME = true;
+          forceSSL = true;
+        };
+      };
+
+      netdata = {
+        enable = true;
+        config = {
+          # web.mode = "none";
+          # web."bind to" = "127.0.0.1:19999";
+          global = {
+            "debug log" = "syslog";
+            "access log" = "syslog";
+            "error log" = "syslog";
+          };
+        };
+      };
+    };
+
+    # nixpkgs.overlays = [
+    #   (final: prev: {
+    #     prometheus-systemd-exporter = prev.prometheus-systemd-exporter.overrideAttrs {
+    #       src = final.fetchFromGitHub {
+    #         owner = "ibizaman";
+    #         repo = prev.prometheus-systemd-exporter.pname;
+    #         # rev = "v${prev.prometheus-systemd-exporter.version}";
+    #         rev = "next_timer";
+    #         sha256 = "sha256-jzkh/616tsJbNxFtZ0xbdBQc16TMIYr9QOkPaeQw8xA=";
+    #       };
+    #
+    #       vendorHash = "sha256-4hsQ1417jLNOAqGkfCkzrmEtYR4YLLW2j0CiJtPg6GI=";
+    #     };
+    #   })
+    # ];
+  };
+}