modules/prometheus: Support prometheus servers in agent mode

1 files changed, 109 insertions, 0 deletions

diff --git a/tests/by-name/mo/monitoring-federation/test.nix b/tests/by-name/mo/monitoring-federation/test.nix
new file mode 100644
index 0000000..a64577d
--- /dev/null
+++ b/tests/by-name/mo/monitoring-federation/test.nix
@@ -0,0 +1,109 @@
+{
+  extraModules,
+  pkgs,
+  vhack,
+  ...
+}: let
+  prometheusMain = "prometheus.server2.server.org";
+  remoteWriteTo = {
+    url = "https://${prometheusMain}";
+  };
+  promPort = 3001;
+in
+  vhack.runTest {
+    name = "monitoring";
+
+    serverDomains = [
+      {server2 = prometheusMain;}
+    ];
+
+    nodes = {
+      server2 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        environment.systemPackages = [
+          pkgs.curl
+        ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              remoteWriteReceiver = prometheusMain;
+            };
+          };
+        };
+      };
+      server3 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              inherit remoteWriteTo;
+            };
+          };
+        };
+      };
+      server4 = {config, ...}: {
+        imports =
+          extraModules
+          ++ [
+            ../../../../modules
+          ];
+
+        vhack = {
+          monitoring = {
+            prometheus = {
+              enable = true;
+              port = promPort;
+              inherit remoteWriteTo;
+            };
+          };
+        };
+      };
+    };
+
+    services = [
+      {server2 = "prometheus.service";}
+      {server3 = "prometheus.service";}
+      {server4 = "prometheus.service";}
+    ];
+
+    testScript = {...}:
+    # Python
+    ''
+      import json, time
+
+      # Give the Prometheus servers some time to generate metrics and sync with the main
+      # one.
+      time.sleep(1)
+
+      metrics = server2.succeed("${pkgs.writeShellScript "query-metrics" ''
+        curl --silent \
+          http://127.0.0.1:${toString promPort}/api/v1/query?query="go_gc_cleanups_queued_cleanups_total"
+      ''}")
+
+      metrics = json.loads(metrics)
+
+      hosts = []
+      for result in metrics["data"]["result"]:
+        server2.log(json.dumps(result,indent=4))
+        hosts.append(result["metric"]["hostname"])
+
+      hosts.sort()
+
+      assert hosts == ["server2", "server3", "server4"], f"Not all hosts used the remote write (got {hosts})"
+    '';
+  }