diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/by-name/mo/monitoring-basic/test.nix (renamed from tests/by-name/mo/monitoring/test.nix) | 1 | ||||
| -rw-r--r-- | tests/by-name/mo/monitoring-federation/test.nix | 109 | ||||
| -rw-r--r-- | tests/common/acme/default.nix | 7 | ||||
| -rw-r--r-- | tests/default.nix | 26 |
4 files changed, 132 insertions, 11 deletions
diff --git a/tests/by-name/mo/monitoring/test.nix b/tests/by-name/mo/monitoring-basic/test.nix index 0b2ed1c..a00331f 100644 --- a/tests/by-name/mo/monitoring/test.nix +++ b/tests/by-name/mo/monitoring-basic/test.nix @@ -1,6 +1,5 @@ { extraModules, - pkgs, vhack, ... }: let diff --git a/tests/by-name/mo/monitoring-federation/test.nix b/tests/by-name/mo/monitoring-federation/test.nix new file mode 100644 index 0000000..a64577d --- /dev/null +++ b/tests/by-name/mo/monitoring-federation/test.nix @@ -0,0 +1,109 @@ +{ + extraModules, + pkgs, + vhack, + ... +}: let + prometheusMain = "prometheus.server2.server.org"; + remoteWriteTo = { + url = "https://${prometheusMain}"; + }; + promPort = 3001; +in + vhack.runTest { + name = "monitoring"; + + serverDomains = [ + {server2 = prometheusMain;} + ]; + + nodes = { + server2 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + environment.systemPackages = [ + pkgs.curl + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + remoteWriteReceiver = prometheusMain; + }; + }; + }; + }; + server3 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + inherit remoteWriteTo; + }; + }; + }; + }; + server4 = {config, ...}: { + imports = + extraModules + ++ [ + ../../../../modules + ]; + + vhack = { + monitoring = { + prometheus = { + enable = true; + port = promPort; + inherit remoteWriteTo; + }; + }; + }; + }; + }; + + services = [ + {server2 = "prometheus.service";} + {server3 = "prometheus.service";} + {server4 = "prometheus.service";} + ]; + + testScript = {...}: + # Python + '' + import json, time + + # Give the Prometheus servers some time to generate metrics and sync with the main + # one. + time.sleep(1) + + metrics = server2.succeed("${pkgs.writeShellScript "query-metrics" '' + curl --silent \ + http://127.0.0.1:${toString promPort}/api/v1/query?query="go_gc_cleanups_queued_cleanups_total" + ''}") + + metrics = json.loads(metrics) + + hosts = [] + for result in metrics["data"]["result"]: + server2.log(json.dumps(result,indent=4)) + hosts.append(result["metric"]["hostname"]) + + hosts.sort() + + assert hosts == ["server2", "server3", "server4"], f"Not all hosts used the remote write (got {hosts})" + ''; + } diff --git a/tests/common/acme/default.nix b/tests/common/acme/default.nix index c756a4f..e7869c2 100644 --- a/tests/common/acme/default.nix +++ b/tests/common/acme/default.nix @@ -29,14 +29,11 @@ in { ( # python '' # Start dependencies for the other services - acme.start() + start_all() + acme.wait_for_unit("pebble.service") - name_server.start() name_server.wait_for_unit("nsd.service") - # Start actual test - start_all() - with subtest("Add pebble ca key to all services"): for node in [name_server, ${builtins.concatStringsSep "," clients}]: node.wait_until_succeeds("curl https://acme.test:15000/roots/0") diff --git a/tests/default.nix b/tests/default.nix index 48fa9ec..9bab516 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -136,21 +136,37 @@ return status == 0 wasAlive: bool + deadCounter: int def run_and_wait(host, unit: str): global wasAlive + global deadCounter + wasAlive = False + deadCounter = 0 + def wait_state(_last_try: bool) -> bool: global wasAlive + global deadCounter state = host.get_unit_property(unit, "SubState") - if state == "failed": - assert False, f'Unit "{unit}" reached state "{state}"' + host.log(f"(waiting) {unit} -> {state}") - if state == "start" or state == "active" or state == "running": + match state: + case "failed": + assert False, f'Unit "{unit}" reached state "{state}"' + case "start" | "active" | "running": wasAlive = True + case "dead": + if wasAlive: + return True + else: + if deadCounter >= 5: + return True - host.log(f"(waiting) {unit} -> {state}") - return wasAlive and state == "dead" + deadCounter = deadCounter + 1 + return False + + return False with host.nested(f"Waiting for unit {unit}"): retry(wait_state) |