modules/prometheus: Support prometheus servers in agent mode

2 files changed, 176 insertions, 1 deletions

diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
index 32477b6..ca947d3 100644
--- a/modules/by-name/mo/monitoring/components/prometheus.nix
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -21,6 +21,35 @@ in {
       default = 3001;
     };
 
+    remoteWriteReceiver = lib.mkOption {
+      example = "https://prometheus.server2.vhack.eu";
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        If not null, this prometheus server will be exposed via a reverse proxy and allow
+        other prometheus servers to write metrics to it.
+      '';
+    };
+    remoteWriteTo = lib.mkOption {
+      example = "https://prometheus.server3.vhack.eu";
+      type = lib.types.nullOr (lib.types.submodule {
+        options = {
+          port = lib.mkOption {
+            type = lib.types.nullOr lib.types.port;
+            default = null;
+          };
+          url = lib.mkOption {
+            type = lib.types.str;
+          };
+        };
+      });
+      default = null;
+      description = ''
+        If not null, this prometheus server will run in agent mode, writing all recived
+        metrics to the specified remote prometheus server.
+      '';
+    };
+
     sources = lib.mkOption {
       type = lib.types.listOf (lib.types.submodule {
         options = {
@@ -29,6 +58,7 @@ in {
           };
           target = lib.mkOption {
             type = lib.types.str;
+            description = "The target url to scrape from.";
           };
           extra = lib.mkOption {
             type = lib.types.attrsOf lib.types.anything;
@@ -67,6 +97,20 @@ in {
       ];
     };
 
+    services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) {
+      virtualHosts.${cfg.remoteWriteReceiver} = {
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write";
+
+          recommendedProxySettings = true;
+          proxyWebsockets = true;
+        };
+
+        enableACME = true;
+        forceSSL = true;
+      };
+    };
+
     users = {
       groups = {
         node-exporter.gid = config.vhack.constants.ids.gids.node-exporter;
@@ -88,6 +132,28 @@ in {
         globalConfig = {
           scrape_interval = "15s";
         };
+        remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [
+          {
+            url = let
+              port =
+                if (cfg.remoteWriteTo.port != null)
+                then ":${toString cfg.remoteWriteTo.port}"
+                else "";
+            in "${cfg.remoteWriteTo.url}${port}";
+            # TODO: Needs RFC42 for prometheus <2026-07-19>
+            # protobuf_message = "io.prometheus.write.v2.Request";
+          }
+        ];
+
+        enableAgentMode = cfg.remoteWriteTo != null;
+        extraFlags = lib.mkMerge [
+          (
+            lib.mkIf (cfg.remoteWriteReceiver != null) [
+              "--web.enable-remote-write-receiver"
+              # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request"
+            ]
+          )
+        ];
 
         scrapeConfigs = builtins.map (attr:
           nixLib.warnMerge {
@@ -99,7 +165,7 @@ in {
               }
             ];
           }
-          attr.extra "prometheus scrapeConfig")
+          attr.extra "prometheus scrapeConfig '${attr.name}'")
         cfg.sources;
         # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
         #     job_name = "redis";
diff --git a/tests/by-name/mo/monitoring-federation/test.nix b/tests/by-name/mo/monitoring-federation/test.nix
new file mode 100644
index 0000000..a64577d
--- /dev/null
+++ b/tests/by-name/mo/monitoring-federation/test.nix
@@ -0,0 +1,109 @@
+{
+  extraModules,
+  pkgs,
+  vhack,
+  ...
+}: let
+  prometheusMain = "prometheus.server2.server.org";
+  remoteWriteTo = {
+    url = "https://${prometheusMain}";
+  };
+  promPort = 3001;
+in
+  vhack.runTest {
+    name = "monitoring";
+
+    serverDomains = [
+      {server2 = prometheusMain;}
+    ];
+
+    nodes = {
+      server2 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        environment.systemPackages = [
+          pkgs.curl
+        ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              remoteWriteReceiver = prometheusMain;
+            };
+          };
+        };
+      };
+      server3 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              inherit remoteWriteTo;
+            };
+          };
+        };
+      };
+      server4 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              inherit remoteWriteTo;
+            };
+          };
+        };
+      };
+    };
+
+    services = [
+      {server2 = "prometheus.service";}
+      {server3 = "prometheus.service";}
+      {server4 = "prometheus.service";}
+    ];
+
+    testScript = {...}:
+    # Python
+    ''
+      import json, time
+
+      # Give the Prometheus servers some time to generate metrics and sync with the main
+      # one.
+      time.sleep(1)
+
+      metrics = server2.succeed("${pkgs.writeShellScript "query-metrics" ''
+        curl --silent \
+          http://127.0.0.1:${toString promPort}/api/v1/query?query="go_gc_cleanups_queued_cleanups_total"
+      ''}")
+
+      metrics = json.loads(metrics)
+
+      hosts = []
+      for result in metrics["data"]["result"]:
+        server2.log(json.dumps(result,indent=4))
+        hosts.append(result["metric"]["hostname"])
+
+      hosts.sort()
+
+      assert hosts == ["server2", "server3", "server4"], f"Not all hosts used the remote write (got {hosts})"
+    '';
+  }