modules/prometheus: Support prometheus servers in agent mode

1 files changed, 67 insertions, 1 deletions

diff --git a/modules/by-name/mo/monitoring/components/prometheus.nix b/modules/by-name/mo/monitoring/components/prometheus.nix
index 32477b6..ca947d3 100644
--- a/modules/by-name/mo/monitoring/components/prometheus.nix
+++ b/modules/by-name/mo/monitoring/components/prometheus.nix
@@ -21,6 +21,35 @@ in {
       default = 3001;
     };
 
+    remoteWriteReceiver = lib.mkOption {
+      example = "https://prometheus.server2.vhack.eu";
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        If not null, this prometheus server will be exposed via a reverse proxy and allow
+        other prometheus servers to write metrics to it.
+      '';
+    };
+    remoteWriteTo = lib.mkOption {
+      example = "https://prometheus.server3.vhack.eu";
+      type = lib.types.nullOr (lib.types.submodule {
+        options = {
+          port = lib.mkOption {
+            type = lib.types.nullOr lib.types.port;
+            default = null;
+          };
+          url = lib.mkOption {
+            type = lib.types.str;
+          };
+        };
+      });
+      default = null;
+      description = ''
+        If not null, this prometheus server will run in agent mode, writing all recived
+        metrics to the specified remote prometheus server.
+      '';
+    };
+
     sources = lib.mkOption {
       type = lib.types.listOf (lib.types.submodule {
         options = {
@@ -29,6 +58,7 @@ in {
           };
           target = lib.mkOption {
             type = lib.types.str;
+            description = "The target url to scrape from.";
           };
           extra = lib.mkOption {
             type = lib.types.attrsOf lib.types.anything;
@@ -67,6 +97,20 @@ in {
       ];
     };
 
+    services.nginx = lib.mkIf (cfg.remoteWriteReceiver != null) {
+      virtualHosts.${cfg.remoteWriteReceiver} = {
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString cfg.port}/api/v1/write";
+
+          recommendedProxySettings = true;
+          proxyWebsockets = true;
+        };
+
+        enableACME = true;
+        forceSSL = true;
+      };
+    };
+
     users = {
       groups = {
         node-exporter.gid = config.vhack.constants.ids.gids.node-exporter;
@@ -88,6 +132,28 @@ in {
         globalConfig = {
           scrape_interval = "15s";
         };
+        remoteWrite = lib.optionals (cfg.remoteWriteTo != null) [
+          {
+            url = let
+              port =
+                if (cfg.remoteWriteTo.port != null)
+                then ":${toString cfg.remoteWriteTo.port}"
+                else "";
+            in "${cfg.remoteWriteTo.url}${port}";
+            # TODO: Needs RFC42 for prometheus <2026-07-19>
+            # protobuf_message = "io.prometheus.write.v2.Request";
+          }
+        ];
+
+        enableAgentMode = cfg.remoteWriteTo != null;
+        extraFlags = lib.mkMerge [
+          (
+            lib.mkIf (cfg.remoteWriteReceiver != null) [
+              "--web.enable-remote-write-receiver"
+              # "--web.remote-write-receiver.accepted-protobuf-messages=io.prometheus.write.v2.Request"
+            ]
+          )
+        ];
 
         scrapeConfigs = builtins.map (attr:
           nixLib.warnMerge {
@@ -99,7 +165,7 @@ in {
               }
             ];
           }
-          attr.extra "prometheus scrapeConfig")
+          attr.extra "prometheus scrapeConfig '${attr.name}'")
         cfg.sources;
         # ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
         #     job_name = "redis";