{ extraModules, pkgs, vhack, ... }: let prometheusMain = "prometheus.server2.server.org"; remoteWriteTo = { url = "https://${prometheusMain}"; }; promPort = 3001; in vhack.runTest { name = "monitoring"; serverDomains = [ {server2 = prometheusMain;} ]; nodes = { server2 = {config, ...}: { imports = extraModules ++ [ ../../../../modules ]; environment.systemPackages = [ pkgs.curl ]; vhack = { monitoring = { prometheus = { enable = true; port = promPort; remoteWriteReceiver = prometheusMain; }; }; }; }; server3 = {config, ...}: { imports = extraModules ++ [ ../../../../modules ]; vhack = { monitoring = { prometheus = { enable = true; port = promPort; inherit remoteWriteTo; }; }; }; }; server4 = {config, ...}: { imports = extraModules ++ [ ../../../../modules ]; vhack = { monitoring = { prometheus = { enable = true; port = promPort; inherit remoteWriteTo; }; }; }; }; }; services = [ {server2 = "prometheus.service";} {server3 = "prometheus.service";} {server4 = "prometheus.service";} ]; testScript = {...}: # Python '' import json, time # Give the Prometheus servers some time to generate metrics and sync with the main # one. time.sleep(1) metrics = server2.succeed("${pkgs.writeShellScript "query-metrics" '' curl --silent \ http://127.0.0.1:${toString promPort}/api/v1/query?query="go_gc_cleanups_queued_cleanups_total" ''}") metrics = json.loads(metrics) hosts = [] for result in metrics["data"]["result"]: server2.log(json.dumps(result,indent=4)) hosts.append(result["metric"]["hostname"]) hosts.sort() assert hosts == ["server2", "server3", "server4"], f"Not all hosts used the remote write (got {hosts})" ''; }